-
Notifications
You must be signed in to change notification settings - Fork 99
/
cp.go
3104 lines (2609 loc) · 114 KB
/
cp.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
package lib
import (
"fmt"
"hash/fnv"
"io/ioutil"
"math"
"net/http"
"os"
"path/filepath"
"strconv"
"strings"
"sync"
"time"
oss "github.com/aliyun/aliyun-oss-go-sdk/oss"
leveldb "github.com/syndtr/goleveldb/leveldb"
)
type operationType int
const (
operationTypePut operationType = iota
operationTypeGet
operationTypeCopy
)
const (
opUpload string = "upload"
opDownload = "download"
opCopy = "copy"
)
/*
* Put same type variables together to make them 64bits alignment to avoid
* atomic.AddInt64() panic
* Please guarantee the alignment if you add new filed
*/
type copyOptionType struct {
cpDir string
snapshotPath string
vrange string
encodingType string
meta string
options []oss.Option
filters []filterOptionType
threshold int64
routines int64
reporter *Reporter
snapshotldb *leveldb.DB
recursive bool
force bool
update bool
ctnu bool
payerOptions []oss.Option
partitionInfo string
partitionIndex int
partitionCount int
versionId string
enableSymlinkDir bool
onlyCurrentDir bool
disableDirObject bool
disableAllSymlink bool
tagging string
opType operationType
bSyncCommand bool
}
type filterOptionType struct {
name string
pattern string
}
type fileInfoType struct {
filePath string
dir string
}
type objectInfoType struct {
prefix string
relativeKey string
size int64
lastModified time.Time
}
var (
mu sync.RWMutex // mu is the mutex for interacting with user
snapmu sync.RWMutex
chProgressSignal chan chProgressSignalType
signalNum = 0
)
type chProgressSignalType struct {
finish bool
exitStat int
}
func freshProgress() {
if len(chProgressSignal) <= signalNum {
chProgressSignal <- chProgressSignalType{false, normalExit}
}
}
// OssProgressListener progress listener
type OssProgressListener struct {
monitor *CPMonitor
lastSize int64
currSize int64
failedEvent bool
}
// ProgressChanged handle progress event
func (l *OssProgressListener) ProgressChanged(event *oss.ProgressEvent) {
if event.EventType == oss.TransferDataEvent {
l.monitor.updateTransferSize(event.RwBytes)
l.monitor.updateDealSize(event.RwBytes)
l.failedEvent = false
} else if !l.failedEvent && event.EventType == oss.TransferFailedEvent {
l.monitor.updateDealSize(-event.ConsumedBytes)
l.failedEvent = true
}
freshProgress()
}
// OssProgressListener resume progress listener
type OssResumeProgressListener struct {
monitor *CPMonitor
lastSize int64
currSize int64
failedEvent bool
breakSizeAdded bool
}
// ProgressChanged handle resume progress event
func (l *OssResumeProgressListener) ProgressChanged(event *oss.ProgressEvent) {
if event.EventType == oss.TransferStartedEvent && !l.breakSizeAdded {
l.monitor.updateDealSize(event.ConsumedBytes)
l.breakSizeAdded = true
} else if event.EventType == oss.TransferDataEvent {
l.monitor.updateTransferSize(event.RwBytes)
l.monitor.updateDealSize(event.RwBytes)
}
freshProgress()
}
var specChineseCopy = SpecText{
synopsisText: "上传,下载或拷贝Objects",
paramText: "src_url dest_url [options]",
syntaxText: `
ossutil cp file_url cloud_url [-r] [-f] [-u] [--enable-symlink-dir] [--disable-all-symlink] [--disable-ignore-error] [--only-current-dir] [--output-dir=odir] [--bigfile-threshold=size] [--checkpoint-dir=cdir] [--snapshot-path=sdir] [--payer requester]
ossutil cp cloud_url file_url [-r] [-f] [-u] [--only-current-dir] [--disable-ignore-error] [--output-dir=odir] [--bigfile-threshold=size] [--checkpoint-dir=cdir] [--range=x-y] [--payer requester] [--version-id versionId]
ossutil cp cloud_url cloud_url [-r] [-f] [-u] [--only-current-dir] [--disable-ignore-error] [--output-dir=odir] [--bigfile-threshold=size] [--checkpoint-dir=cdir] [--payer requester] [--version-id versionId]
`,
detailHelpText: `
该命令允许:从本地文件系统上传文件到oss,从oss下载object到本地文件系统,在oss
上进行object拷贝。分别对应下述三种操作:
ossutil cp file_url oss://bucket[/prefix] [-r] [-f] [-u] [--output-dir=odir] [--bigfile-threshold=size] [--checkpoint-dir=file] [--snapshot-path=sdir] [--payer requester]
ossutil cp oss://bucket[/prefix] file_url [-r] [-f] [-u] [--output-dir=odir] [--bigfile-threshold=size] [--checkpoint-dir=file] [--range=x-y] [--payer requester]
ossutil cp oss://src_bucket[/src_prefix] oss://dest_bucket[/dest_prefix] [-r] [-f] [-u] [--output-dir=odir] [--bigfile-threshold=size] [--checkpoint-dir=file] [--payer requester]
其中file_url代表本地文件系统中的文件路径,支持相对路径或绝对路径,请遵循本地文
件系统的使用格式;
oss://bucket[/prefix]代表oss上的object,支持前缀匹配,不支持通配符。
ossutil通过oss://前缀区分本地文件系统的文件和oss文件。
注意:在oss间拷贝文件,目前只支持拷贝object,不支持拷贝未complete的Multipart。
--recursive选项
(1)上传文件到oss时,如果file_url为目录,则必须指定--recursive选项,否则无需指
定--recursive选项。
(2)从oss下载或在oss间拷贝文件时:
如果未指定--recursive选项,则认为拷贝单个object,此时请确保src_url精确指定待
拷贝的object,如果object不存在,则报错。
如果指定了--recursive选项,ossutil会对src_url进行prefix匹配查找,对这些objects
批量拷贝,如果拷贝失败,已经执行的拷贝不会回退。
在进行批量文件上传(或下载、拷贝)时,如果其中某个文件操作失败,ossutil不会退出,
而是继续进行其他文件的上传(或下载、拷贝)动作,并将出错文件的错误信息记录到report
文件中。成功上传(或下载、拷贝)的文件信息将不会被记录到report文件中。
注意:批量操作出错时不继续运行,而是终止运行的情况:
(1)如果未进入批量文件迭代过程,错误已经发生,则不会产生report文件,ossutil会终止
运行,不继续迭代过程。如,用户输入cp命令出错时,不会产生report文件,而是屏幕输出错
误并退出。
(2)如果批量操作过程某文件发生的错误为:Bucket不存在、accessKeyID/accessKeySecret
错误造成的权限验证非法等错误,ossutil会屏幕输出错误并退出。
report文件名为:` + ReportPrefix + `日期_时间` + ReportSuffix + `。report文件是ossutil输出文件的一种,
被放置在ossutil的输出目录下,该目录的路径可以用配置文件中的outputDir选项或命令行
--output-dir选项指定,如果未指定,会使用默认的输出目录:当前目录下的` + DefaultOutputDir + `目录。
注意:ossutil不做report文件的维护工作,请自行查看及清理用户的report文件,避免产生
过多的report文件。
--include和--exclude选项
当指定--recursive选项时,可以指定该选项以指定规则筛选要操作的文件/object。默认情况下,
指定目录中所有项都包含在cp操作中。
规则支持以下格式:
*:匹配索引
?:匹配单个字符
[sequence]:匹配sequence的任意字符
[!sequence]:匹配不在sequence的任意字符
注意:规则不支持带目录的格式,e.g.,--include "/usr/*/test/*.jpg"。
--include和--exclude可以出现多次。当多个规则出现时,这些规则按从左往右的顺序应用。例如:
当前目录下包含3个文件:
testfile1.jpg
testfiel2.txt
testfile33.jpg
$ ossutil cp . oss://my-bucket/path --exclude '*.jpg'
上传testfile2.txt到oss://my-bucket/path/testfile2.txt
$ ossutil cp . oss://my-bucket/path --exclude '*.jpg' --include 'testfile*.jpg'
上传testfile1.jpg到oss://my-bucket/path/testfile1.jpg
上传testfile33.jpg到oss://my-bucket/path/testfile33.jpg
上传testfile2.txt到oss://my-bucket/path/testfile2.txt
$ ossutil cp . oss://my-bucket/path --exclude '*.jpg' --include 'testfile*.jpg' --exclude 'testfile?.jpg'
上传testfile2.txt到oss://my-bucket/path/testfile2.txt
上传testfile33.jpg到oss://my-bucket/path/testfile33.jpg
--meta选项
该选项在上传文件的同时设置object的meta信息。当指定--recursive选项时,会设置所有上传的
objects的meta信息。
可选的header列表如下:
` + formatHeaderString(headerOptionMap, "\n ") + `
以及以` + oss.HTTPHeaderOssMetaPrefix + `开头的header
注意:header不区分大小写,但value区分大小写。设置后将用指定的meta代替原来的meta。没有指定的
HTTP HEADER将保留,没有指定的user meta将会被删除。
--tagging选项
该选项在上传文件的同时设置object的tagging信息。当指定--recursive选项时,会设置所有上传的
objects的tagging信息。
如果一次设置多个tagging,必须使用双引号,比如 "tagA=A&tagB=B"
--acl选项
该选项在上传文件的同时设置object的acl信息。当指定--recursive选项时,会设置所有上传的
objects的acl信息。
object的acl有四种:
` + formatACLString(objectACL, "\n ") + `
acl的详细信息请参见:https://help.aliyun.com/document_detail/31867.html
--output-dir选项
该选项指定ossutil输出文件存放的目录,默认为:当前目录下的` + DefaultOutputDir + `目录。如果指定
的目录不存在,ossutil会自动创建该目录,如果用户指定的路径已存在并且不是目录,会报错。
输出文件表示ossutil在运行过程中产生的输出文件,目前包含:在cp命令中ossutil运行出错时
产生的report文件,在批量set-acl、set-meta、restore出错时产生的report文件。
增量上传/下载/拷贝:
--update选项(-u)
如果指定了该选项,ossutil只有当目标文件(或object)不存在,或源文件(或object)新于
目标文件(或object)时,才执行上传、下载、拷贝。当指定了该选项时,无论--force选项是
否指定了,在目标文件存在时,ossutil都不会提示,直接采取上述策略。
该选项可用于当批量拷贝失败时,重传时跳过已经成功的文件。实现增量上传。
--snapshot-path选项
该选项用于在某些场景下加速增量上传批量文件(目前,下载和拷贝不支持该选项)。此场景为:
文件数较多且两次上传期间没有其他用户更改了oss上的对应object。
在cp上传文件时使用该选项,ossutil在指定的目录下生成文件记录文件上传的快照信息,在下一
次指定该选项上传时,ossutil会读取指定路径下的快照信息进行增量上传。用户指定的snapshot-path
必须为本地文件系统上的可写目录,若该路径目录不存在,ossutil会创建该文件用于记录快照信息,
如果该路径文件已存在,ossutil会读取里面的快照信息,根据快照信息进行增量上传(只上传上次
未成功上传的文件和本地进行过修改的文件),并更新快照信息。
注意:
(1)因为该命令通过在本地记录成功上传的文件的本地lastModifiedTime,从而在下次上传时通过
比较lastModifiedTime来决定是否跳过相同文件的上传,所以在使用该选项时,请确保两次上传期
间没有其他用户更改了oss上的对应object。当不满足该场景时,如果想要增量上传批量文件,请使
用--update选项。
(2)ossutil不会主动删除snapshot-path下的快照信息,为了避免快照信息过多,当用户确定快照信
息无用时,请用户自行清理snapshot-path。
(3)由于读写snapshot信息需要额外开销,当要批量上传的文件数比较少或网络状况比较好或有其
他用户操作相同object时,并不建议使用该选项。可以使用--update选项来增量上传。
注意:--update选项和--snapshot-path选项可以同时使用,ossutil会优先根据snapshot-path信息判断
是否跳过上传,如果不满足跳过条件,再根据--update判断是否跳过上传。如果指定了这两种增量上
传策略之中的任何一种,ossutil将根据策略判断是否进行上传/下载/拷贝,当遇到目标端的文件已
存在,也不会询问用户是否进行替换操作,此时--force选项不再生效。
另外,增量下载策略不会考虑--range选项的值,即增量下载策略只参考文件是否存在和lastModifiedTime
信息来决定,即如果满足跳过下载的条件,就算两次下载指定的range不一样,也同样会跳过文件。
所以请避免两者共同使用!
其他选项:
--force选项
如果dest_url指定的文件或objects在oss上已经存在,并且未指定--update或--snapshot-path选项,
ossutil会询问是否进行替换操作(输入非法时默认不替换),如果指定了--force选项,则不询问,
强制替换。该选项只有在未指定--update或--snapshot-path选项时有效,否则按指定的选项操作。
--range选项
如果下载文件时只需要下载文件内容的部分,可以通过--range选项来指定下载的文件内容范围,如
果指定了该选项,则大文件的多线程下载和断点续传默认无效。
文件偏移从0开始,有三种形式:0-9或3-或-9。
比如指定--range=0-9,表示下载指定文件的第0到第9这10个字符;
指定--range=3-,表示下载指定文件第3字符到文件结尾的内容;
指定--range=-9,表示下载指定文件结尾的9个字符。
如果指定的范围超过文件长度范围,会下载整个文件。
关于range的更多信息见:https://help.aliyun.com/document_detail/31980.html?spm=5176.doc31994.6.860.YH7LL1
如果想下载整个文件请不要指定这个选项。
目前上传和拷贝文件,不支持--range选项。
注意:指定了增量下载策略时(-u选项),策略决定是否跳过下载不会考虑range范围是否变化,即
使前后几次下载range范围不一样,满足增量下载条件时,ossutil同样会跳过下载,所以请避免两者
同时使用!
--encoding-type选项
如果指定该选项为url,则表示输入的object名和文件名都是经过url编码的。
--enable-symlink-dir选项
允许传输链接子目录下文件,如果存在死循环链接文件或者目录,会导致错误,使用前建议用probe命令
检测是否存在死循环链接文件或者目录
--disable-all-symlink选项
上传目录时,忽略掉该目录下所有的链接文件以及链接子目录
--only-current-dir
和-r选项一起使用,表示只操作当前目录下的文件, 会忽略当前目录下的子目录, 如果是下载或者拷贝oss
的目录,目录后面要加上反斜线/
大文件断点续传:
如果源文件大小超过--bigfile-threshold选项指定的大小(默认为100M),ossutil会认为该文件
为大文件,并自动使用断点续传策略,策略如下:
(1)上传到oss时:ossutil会对大文件自动分片,进行multipart分片上传,如果上传失败,会
在本地的.ossutil_checkpoint目录记录失败信息,下次重传时会读取.ossutil_checkpoint目
录中的信息进行断点续传,当上传成功时会删除.ossutil_checkpoint目录。
(2)从oss下载时:ossutil会自动对大文件分片下载,组装成一个文件,如果下载失败,同样会
在.ossutil_checkpoint目录记录失败信息,重试成功后会删除.ossutil_checkpoint目录。
(3)在oss间拷贝:ossutil会自动对大文件分片,使用Upload Part Copy方式拷贝,同样会在
.ossutil_checkpoint目录记录失败信息,重试成功后会删除.ossutil_checkpoint目录。
注意:
1)小文件不会采用断点续传策略,失败后下次直接重传。
2)在操作(1)和(3)中,如果操作失败,oss上可能会产生未complete的uploadId,但是只要最
终操作成功,就不会存在未complete的uploadId(被组装成object)。
3)上传到oss时,如果.ossutil_checkpoint目录包含在file_url中,.ossutil_checkpoint目录不会
被上传到oss。该目录路径可以用--checkpoint-dir选项指定,如果指定了该选项,请确保指定的目录
可以被删除。
4)如果使用rm命令删除了未complete的Multipart Upload,可能会造成下次cp命令断点续传失败(报
错:NoSuchUpload),这种时候如果想要重新上传整个文件,请删除相应的checkpoint文件。
性能调优:
--jobs选项和--parallel选项(并发控制)
--jobs选项控制多个文件上传/下载/拷贝时,文件间启动的并发数,--parallel控制上传/下载/拷
贝大文件时,分片间的并发数。默认情况下,ossutil会根据文件大小来计算parallel个数(该选项
对于小文件不起作用,进行分片上传/下载/拷贝的大文件文件阈值可由--bigfile-threshold选项来
控制),当进行批量大文件的上传/下载/拷贝时,实际的并发数为jobs个数乘以parallel个数。该
两个选项可由用户调整,当ossutil自行设置的默认并发达不到用户的性能需求时,用户可以自行调
整该两个选项来升降性能。
注意:
1)如果并发数调得太大,由于线程间资源切换及抢夺等,ossutil上传/下载/拷贝性能可能会下降,
所以请根据实际的机器情况调整这两个选项的数值,如果要进行压测,可以一开始将两个数值调低,
慢慢调大寻找最优值。
2)如果--jobs选项和--parallel选项值太大,在机器资源有限的情况下,可能会因为网络传输太慢,
产生EOF错误,这个时候请适当降低--jobs选项和--parallel选项值。
--part-size选项
该选项设置大文件分片上传/下载/拷贝时,每个分片的大小。默认情况下,不需要设置该值,ossutil
会根据文件大小自行决定分片大小和分片并发,当用户上传/下载/拷贝性能达不到需求时,或有其他
特殊需求时,可以设置这些选项。
如果设置了该选项(分片大小),分片个数为:向上取整(文件大小/分片大小),注意如果--parallel
选项值大于分片个数,则多余的parallel不起作用,实际的并发数为分片个数。
如果将part size值设置得过小,可能会影响ossutil文件上传/下载/拷贝的性能,设置得过大,会影
响实际起作用的分片并发数,所以请合理设置part size选项值。
批量文件迁移:
ossutil支持批量文件迁移,在这种场景下,通常的使用方式是:
(1)批量上传:
ossutil cp your_dir oss://your_bucket -r -f -u
ossutil cp your_dir oss://your_bucket -r -f -u --include "*.jpg"
ossutil cp your_dir oss://your_bucket -r -f -u --exclude "*.html"
(2)批量下载:
ossutil cp oss://your_bucket your_dir -r -f -u
ossutil cp oss://your_bucket your_dir -r -f -u --include "*.jpg"
(3)同region的Bucket间迁移:
ossutil cp oss://your_src_bucket oss://your_dest_bucket -r -f -u
具体每个选项的意义,请见上文帮助。
在运行完一轮文件迁移后,请根据屏幕提示查看report文件,处理出错文件。
在批量上传时,如果文件数比较多且没有其他用户操作相同object时,可以使用--snapshot-path选项
进行额外的增量上传加速,更多信息参考上文关于--snapshot-path选项的介绍。命令为:
ossutil cp your_dir oss://your_bucket -r -f -u --shapshot-path=your-path
用法:
该命令有三种用法:
1) ossutil cp file_url oss://bucket[/prefix] [-r] [-f] [-u] [--output-dir=odir] [--bigfile-threshold=size] [--checkpoint-dir=file] [--snapshot-path=sdir]
该用法上传本地文件系统中文件或目录到oss。file_url可以为文件或目录。当file_url为文件
时,无论是否指定--recursive选项都不会影响结果。当file_url为目录时,即使目录为空或者只含
有一个文件,也必须使用--recursive选项,注意,此时ossutil会将file_url下的文件或子目录上传
到oss,但不同于shell拷贝,file_url所代表的首层目录不会被创建。
object命名规则:
当file_url为文件时,如果prefix为空或以/结尾,object名为:dest_url+文件名。
否则,object名为:dest_url。
当file_url为目录时,如果prefix为空或以/结尾,object名为:dest_url+文件或子目录相对
file_url的路径。
否则,object名为:dest_url+/+文件或子目录相对file_url的路径。
2) ossutil cp oss://bucket[/prefix] file_url [-r] [-f] [-u] [--output-dir=odir] [--bigfile-threshold=size] [--checkpoint-dir=file] [--range=x-y]
该用法下载oss上的单个或多个Object到本地文件系统。如果未指定--recursive选项,则ossutil
认为src_url精确指定了待拷贝的单个object,此时不支持prefix匹配,如果object不存在则报错。如
果指定了--recursive选项,ossutil会搜索prefix匹配的objects,批量拷贝这些objects,此时file_url
必须为目录,如果该目录不存在,ossutil自动创建该目录。
文件命名规则:
当file_url为文件时,下载到file_url指定的文件,文件名与file_url保持一致。
当file_url为目录时,下载到file_url指定的目录中,文件名为:object名称,当object名称中含有/或\\时,会创建相应子目录。
注意:对于以/结尾且大小为0的object,会在本地文件系统创建一个目录,而不是尝试创建一个文件。
对于其他object会尝试创建文件。
3) ossutil cp oss://src_bucket[/src_prefix] oss://dest_bucket[/dest_prefix] [-r] [-f] [-u] [--output-dir=odir] [--bigfile-threshold=size] [--checkpoint-dir=file]
该用法在oss间进行object的拷贝。其中src_bucket与dest_bucket可以相同,注意,当src_url与
dest_url完全相同时,ossutil不会做任何事情,直接提示退出(除非指定--meta选项)。如果未指定
--recursive选项,则认为ossutil会搜索prefix匹配的objects,批量拷贝这些objects。
注意:批量拷贝时,src_url包含dest_url,或dest_url包含src_url是不允许的(dest_url以src_url为
前缀时,会产生递归拷贝,src_url以dest_url为前缀时,会覆盖待拷贝文件)。单个拷贝时,该情况是
允许的。
object命名规则:
当src_url为单个文件时,如果dest_url的prefix为空或以/结尾,object名为:dest_url+object名去除所在父目录的路径。
否则,object名为:dest_url。
当src_url为多个文件时,object名为:dest_url+源object名去除src_prefix。
以上三种用法中如果指定了--recursive选项,均可以使用--include或--exclude选项使用通配符的方式过滤要操作的文件。
`,
sampleText: `
1) 上传文件到oss
假设本地local_dir目录中有文件a,b,目录c和d,目录c为空,目录d中包含文件dd。
ossutil cp local_dir/a oss://bucket1
生成:
oss://bucket1/a
ossutil cp local_dir/a oss://bucket1/b
生成:
oss://bucket1/b
ossutil cp local_dir/a oss://bucket1/b/
生成:
oss://bucket1/b/a
ossutil cp local_dir oss://bucket1/b/
报错
ossutil cp local_dir oss://bucket1/b -r
生成:
oss://bucket1/b/a
oss://bucket1/b/b
oss://bucket1/b/c/
oss://bucket1/b/d/
oss://bucket1/b/d/dd
ossutil cp local_dir oss://bucket1/b -r
如果某文件上传发生服务器内部错误等失败,会在当前目录下的ossutil_output目录中产生report文件
记录错误信息,并尝试其他文件的上传操作。
ossutil cp local_dir oss://bucket1/b -r --output-dir=your_dir
如果某文件上传发生服务器内部错误等失败,会在your_dir中产生report文件记录错误信息,并尝试其
他文件的上传操作。
ossutil cp local_dir oss://bucket1/b -r -u
使用--update策略进行增量上传
ossutil cp local_dir oss://bucket1/b -r --snapshot-path=your_local_path
使用--snapshot-path策略进行增量上传
ossutil cp local_dir oss://bucket1/b -r -u --snapshot-path=your_local_path
同时使用--snapshot-path和--update策略进行增量上传
ossutil cp %e4%b8%ad%e6%96%87 oss://bucket1/%e6%b5%8b%e8%af%95 --encoding-type url
在本地查找文件名为“中文”的文件,并上传到bucket1生成名称为”测试“的object
ossutil cp local_dir oss://bucket1/b -r --enable-symlink-dir
支持上传符号链接子目录下的文件
ossutil cp local_dir oss://bucket1/b -r --only-current-dir
只上传当前目录的下文件,忽略其他的子目录
ossutil cp local_dir oss://bucket1/b -r --disable-all-symlink
忽略所有的链接子文件以及链接子目录
ossutil cp local_dir oss://bucket1/b --tagging "tagA=A&tagB=B" -r
上传的同时设置两个tagging,key分别为tagA和tagB,value分别为A和B
2) 从oss下载object
假设oss上有下列objects:
oss://bucket/abcdir1/a
oss://bucket/abcdir1/b
oss://bucket/abcdir1/c
oss://bucket/abcdir2/a/
oss://bucket/abcdir2/b/e
其中oss://bucket/abcdir2/a/的size为0。
ossutil cp oss://bucket/abcdir1/a b
生成文件b
ossutil cp oss://bucket/abcdir1/a b --update
如果文件b已存在,且更新时间不晚于oss://bucket/abcdir1/a,则跳过本次操作。
ossutil cp oss://bucket/abcdir1/a b/
在目录b下生成文件a
ossutil cp oss://bucket/abcdir1/a b/ --range=30-90
在目录b下生成文件a,内容为object:abcdir1/a的第30到第90个字符
ossutil cp oss://bucket/abcdir2/a/ b
如果b为已存在文件,报错。
如果b为已存在目录,在目录b下生成目录a
ossutil cp oss://bucket/abc b
报错,object不存在。
ossutil cp oss://bucket/abc b -r
如果b为已存在文件,报错
否则在目录b下生成目录abcdir1和abcdir2,
目录abcdir1中生成文件a,b,c
目录abcdir2中生成目录a和b,目录b中生成文件e
ossutil cp oss://bucket/ local_dir -r
如果某文件下载发生服务器内部错误等失败,会在当前目录下的ossutil_output目录中产生report文件
记录错误信息,并尝试其他文件的下载操作。
ossutil cp oss://bucket/ local_dir -r --output-dir=your_dir
如果某文件下载发生服务器内部错误等失败,会在your_dir中产生report文件记录错误信息,并尝试其
他文件的下载操作。
ossutil cp oss://bucket/ local_dir -r -u
使用--update策略进行增量下载
ossutil cp oss://bucket1/%e6%b5%8b%e8%af%95 %e4%b8%ad%e6%96%87 --encoding-type url
下载bucket1中名称为”测试“的object到本地,生成文件名为“中文”的文件
ossutil cp oss://bucket/object local_file --version-id versionId
指定object版本下载
ossutil cp oss://bucket/dir/ local_dir -r --only-current-dir
只下载当前目录下的object, 忽略其他子目录
3) 在oss间拷贝
假设oss上有下列objects:
oss://bucket/abcdir1/a
oss://bucket/abcdir1/b
oss://bucket/abcdir1/c
oss://bucket/abcdir2/a/
oss://bucket/abcdir2/b/e
ossutil cp oss://bucket/abcdir1/a oss://bucket1
生成:
oss://bucket1/a
ossutil cp oss://bucket/abcdir1/a oss://bucket1/b
生成:
oss://bucket1/b
ossutil cp oss://bucket/abcdir1/a oss://bucket/abcdir1/a/
生成:
oss://bucket/abcdir1/a/a
ossutil cp oss://bucket/abcdir1/a/ oss://bucket/abcdir1/b/
生成:
oss://bucket/abcdir1/b/a/
ossutil cp oss://bucket/abcdir1/a oss://bucket/abcdir1/a/ -r
报错,递归拷贝
ossutil cp oss://bucket/abcdir1/a oss://bucket1/b/
生成:
oss://bucket1/b/a
ossutil cp oss://bucket/abc oss://bucket1/b/
报错,object不存在
ossutil cp oss://bucket/abc oss://bucket1/123 -r
生成:
oss://bucket1/123dir1/a
oss://bucket1/123dir1/b
oss://bucket1/123dir1/c
oss://bucket1/123dir2/a/
oss://bucket1/123dir2/b/e
ossutil cp oss://bucket/abc oss://bucket1/123/ -r
生成:
oss://bucket1/123/dir1/a
oss://bucket1/123/dir1/b
oss://bucket1/123/dir1/c
oss://bucket1/123/dir2/a/
oss://bucket1/123/dir2/b/e
ossutil cp oss://bucket/abcdir1/a oss://bucket1/ -r
报错,因为此时目标object名称为空,非法
ossutil cp oss://bucket/ oss://bucket1/ -r
如果某文件拷贝发生服务器内部错误等失败,会在当前目录下的ossutil_output目录中产生report文件
记录错误信息,并尝试其他文件的拷贝操作。
ossutil cp oss://bucket/ oss://bucket1/ -r --output-dir=your_dir
如果某文件拷贝发生服务器内部错误等失败,会在your_dir中产生report文件记录错误文件的信息,并
尝试其他文件的拷贝操作。
ossutil cp oss://bucket/ oss://bucket1/ -r -u
使用--update策略进行增量拷贝
ossutil cp oss://bucket1/%e6%b5%8b%e8%af%95 oss://bucket2/%e4%b8%ad%e6%96%87 --encoding-type url
拷贝bucket1中名称为”测试“的object到bucket2,生成object名为“中文”的object
ossutil cp oss://bucket/object1 oss://bucket/object2 --version-id versionId
指定object版本copy
ossutil cp oss://bucket/dir/ oss://bucket1/ -r --only-current-dir
只copy当前目录下的object, 忽略其他子目录
ossutil cp oss://bucket/object1 oss://bucket/object2 --tagging "tagA=A&tagB=B"
copy的同时设置两个tagging,key分别为tagA和tagB,value分别为A和B
`,
}
var specEnglishCopy = SpecText{
synopsisText: "Upload, Download or Copy Objects",
paramText: "src_url dest_url [options]",
syntaxText: `
ossutil cp file_url cloud_url [-r] [-f] [-u] [--enable-symlink-dir] [--disable-all-symlink] [--disable-ignore-error] [--only-current-dir] [--output-dir=odir] [--bigfile-threshold=size] [--checkpoint-dir=cdir] [--snapshot-path=sdir] [--payer requester]
ossutil cp cloud_url file_url [-r] [-f] [-u] [--only-current-dir] [--output-dir=odir] [--disable-ignore-error] [--bigfile-threshold=size] [--checkpoint-dir=cdir] [--range=x-y] [--payer requester]
ossutil cp cloud_url cloud_url [-r] [-f] [-u] [--only-current-dir] [--output-dir=odir] [--disable-ignore-error] [--bigfile-threshold=size] [--checkpoint-dir=cdir] [--payer requester]
`,
detailHelpText: `
The command allows:
1. Upload file from local file system to oss
2. Download object from oss to local file system
3. Copy objects between oss
Which matches with the following three kinds of operations:
ossutil cp file_url oss://bucket[/prefix] [-r] [-f] [-u] [--output-dir=odir] [--bigfile-threshold=size] [--checkpoint-dir=file] [--snapshot-path=sdir] [--payer requester]
ossutil cp oss://bucket[/prefix] file_url [-r] [-f] [-u] [--output-dir=odir] [--bigfile-threshold=size] [--checkpoint-dir=file] [--range=x-y] [--payer requester]
ossutil cp oss://src_bucket[/src_prefix] oss://dest_bucket[/dest_prefix] [-r] [-f] [-u] [--output-dir=odir] [--bigfile-threshold=size] [--checkpoint-dir=file] [--payer requester]
file_url means the file in local file system, it supports relative path and absolute
path, the usage of file_url is same with your local file system. oss://bucket[/prefix]
means object in oss, it supports prefix matching, but not support wildcard.
ossutil sperate file of local system and oss objects by the prefix of oss://, which means
if the url starts with oss://, ossutil considers it as object, else, ossutil considers it
as file in local system.
Note: when copy between oss, ossutil only support copy objects, the uncompleted Multipart
Uploads are not supported.
--recursive option:
(1) Upload file to oss: if file_url is directory, the --recursive option must be specified.
(2) When download objects or copy objects between oss:
If --recursive option is not specified, ossutil download or copy the specified single
object, in the usage, please make sure url exactly specified the object you want to set meta
on, if object not exist, error occurs.
If --recursive option is specified, ossutil will search for prefix-matching objects and
download or copy these objects. If error occurs, objects which has been download or copyed
will not rollback.
By default, if an error occurs to a file in batch upload(/download/copy) files operation,
ossutil will continue to attempt to copy the remaining files, and ossutil will record the
error message to report file. The files succeed copied will not be recorded to report file.
Note: Ossutil will print error information and exit, instead of continue to run if an error
occurs in batch upload(/download/copy) files operation in several situations:
(1) If the error occurs before of entering the upload(/download/copy) iteration, ossutil will
print error message and return, and the report file will not be generated. eg. user enter
an invalid cp command.
(2) If the error occurs during upload(/download/copy) iteration is: NoSuchBucket, AccessDenied
caused by unauthorized authentication and other errors. ossutil will print error message
and return, the report file that has been generated will not be deleted.
Report file name is: ` + ReportPrefix + `Date_Time` + ReportSuffix + `. Report file is one kind
of output files, and will be putted in output directory, the directory can be specified by
--output-dir option or outputDir option in config file. If it's not specified, ossutil will use
the default directory: ` + DefaultOutputDir + ` in current directory.
Note: ossutil will not mainten the report file, please check and clear your output directory
regularlly to avoid too many report files in your output directory.
--include and --exclude option:
When --recursive is specified, these parameters perform pattern matching to either exclude or
include a particular file or object. By default, all files/objects are included.
The following pattern symbols are supported.
*: Matches everything
?: Matches any single character
[sequence]: Matches any character in sequence
[!sequence]: Matches any character not in sequence
Note: does not support patterns containing directory info. e.g., --include "/usr/*/test/*.jpg"
Any number of these parameters can be passed to a command. You can do this by providing an --exclude
or --include argument multiple times, e.g.,
--include "*.txt" --include "*.png".
When there are multi filters, the rule is the filters that appear later in the command take precedence
over filters that appear earlier in the command. e.g.,
--exclude "*" --include "*.txt"
All files will be excluded from the command except for files ending with .txt
--include "*.txt" --exclude "*"
All files will be excluded from the command.
e.g., 3 files in current dir
testfile1.jpg
testfiel2.txt
testfile33.jpg
$ ossutil cp . oss://my-bucket/path --exclude '*.jpg'
upload testfile2.txt to oss://my-bucket/path/testfile2.txt
$ ossutil cp . oss://my-bucket/path --exclude '*.jpg' --include 'testfile*.jpg'
upload testfile1.jpg to oss://my-bucket/path/testfile1.jpg
upload testfile33.jpg to oss://my-bucket/path/testfile33.jpg
upload testfile2.txt to oss://my-bucket/path/testfile2.txt
$ ossutil cp . oss://my-bucket/path --exclude '*.jpg' --include 'testfile*.jpg' --exclude 'testfile?.jpg'
upload testfile2.txt to oss://my-bucket/path/testfile2.txt
upload testfile33.jpg to oss://my-bucket/path/testfile33.jpg
--meta option
This option will set the specified objects' meta data. If --recursive option is specified,
ossutil will set meta for all uploaded objects.
ossutil supports following headers:
` + formatHeaderString(headerOptionMap, "\n ") + `
and headers starts with: ` + oss.HTTPHeaderOssMetaPrefix + `
Note: headers are case-insensitive, but value are case-sensitive. After setting, origin meta will be
replaced with specified meta. HTTP HEADER will be reserved if no speified value. User meta will be
deleted if no specified value.
--tagging option
This option will set the specified objects' tagging data. If --recursive option is specified,
ossutil will set tagging for all uploaded objects.
If you set more than one tagging at a time, you must use double quotes, such as "tagA=A&tagB=B"
--acl option
This option will set acl on the specified objects. If --recursive option is specified,
ossutil will set acl for all uploaded objects.
ossutil support following objet acls:
` + formatACLString(objectACL, "\n ") + `
More information about acl see ACL Control in https://help.aliyun.com/document_detail/31867.html?spm=5176.doc31960.6.147.8dVwsh.
--output-dir option
The option specify the directory to deposit output file generated by ossutil, the default value
is: ` + DefaultOutputDir + ` in current directory. If the directory specified not exist, ossutil will
create the directory automatically, if it exists but is not a directory, ossutil will return an
error.
Output file contains: report file which used to record error message generated by cp command,
set-acl, set-meta, restore command.
Incremental Upload/Download/Copy:
--update option(-u)
Use the --update option to copy only when the source file is newer than the destination file
when the destination file is missing. If --update option is specified, when the destionation
file is existed, ossutil will not prompt and copy when newer, no matter if --force option is
specified or not.
The option can be used when batch copy failed, skip the succeed files in retry.
--snapshot-path option
This option is used to accelerate the incremental upload of batch files in certain scenarios(
currently, download and copy do not support this option). The scenarios is: lots of files and
no other user updated the corresponding object in oss during the two uploads.
If you use the option when batch copy files, ossutil will generate files to record the snapshot
information in the specified directory. When the next time you upload files with the option,
ossutil will read the snapshot information under the specified directory for incremental upload.
The snapshot-path you specified must be a local file system directory can be written in, if the
directory does not exist, ossutil creates the files for recording snapshot information, else
ossutil will read snapshot information from the directory for incremental upload(ossutil will
only upload the files which has not been successfully upload to oss and the files has been locally
modified), and update the snapshot information to the directory.
Note:
(1) The option record the lastModifiedTime of local files which has been successfully upload in
local file system, and compare the lastModifiedTime of local files in the next cp to decided
whether to skip the upload of the files, so if you use the option to achieve incremental upload,
please make sure no other user updated the corresponding object in oss during the two uploads.
If you can not guarantee the scenarios, please use --update option to achieve incremental upload.
(2) Ossutil does not automatically delete snapshot-path snapshot information, in order to avoid too
much snapshot information, when the snapshot information is useless, please clean up your own
snapshot-path on your own.
(3) Due to the extra cost of reading and writing snapshot information, if the file num is not very big,
or the network condition is good, or there may be some other users to modify the corresponding
object in oss during the two uploads, it's not suggested to use the option. you can use --update
option for incremental upload.
Note: --update option and --snapshot-path can be used together, ossutil priority will be based on snapshot
information to determine whether to skip upload, if not satisfied, ossutil will then based on --update
to determine whether to skip upload. If any of those two policies is specified, ossutil will ingnore
--force option, which means whether or not the destionation file exists, ossutil will not ask user
whether to replace the file, and determine whether to upload according to incremental upload policies.
Incremental download will not consider the value of --range option, and only consider whether file
exists and lastModifiedTime. Which means even if the range changs between two download, ossutil will
skip the files which satisfy the incremental download condition, so, please avoid to use both!
Other Options:
--force option
If the file dest_url specified is existed, and --update and --snapshot-path option is not specified,
ossutil will ask if replace the file(if the input is invalid, the file will not be replaced). If
--force option is specified here, ossutil will not prompt, replace by force. The option is useful
only when --update and --snapshot-path option is not specified.
--range option
If user need to range download a file, we can use --range option, if we use the option, then
resume copy of big file and multi-thread copy is ineffective.
The offset of file is start
with 0, there are three forms: 0-9 or 3- or -9.
eg: --range=0-9, means download the first to the tenth character of the file.
--range=3-, means download the fourth character to the end of the file.
--range=-9, means download the last nine character of the file.
If the range exceed the file actual scope, will download the whole file.
More information about range see: https://help.aliyun.com/document_detail/31980.html?spm=5176.doc31994.6.860.YH7LL1
If you need to download the whole file, please do not specify the option.
The option is not supported for upload and copy files.
Note: Incremental download(-u option) will not conside --range option. Which means even if the
range changs between two download, ossutil will skip the files which satisfy the incremental
download condition, so, please avoid to use both!
--encoding-type option
If the --encoding-type option is setted to url, it means the object name and file name are url
endcoded.
--enable-symlink-dir option
Allows transfer of files in the link subdirectory. If there is an infinite loop link file or directory,
it will cause an error.
It is recommended to use the probe command to detect the existence of an infinite loop link file or
directory before use
--disable-all-symlink option
specifies that uploading of symlink files and symlink directories under the directory is not allowed
--only-current-dir
Used with the -r option, it means that only the files in the current directory will be manipulated,
and the subdirectories under the current directory will be ignored.
If you are downloading or copying the oss directory, add a backslash(/) after the directory.
Resume copy of big file:
If the size of source file is bigger than what --bigfile-threshold option specified(default:
100M), ossutil will consider the file as a big file, and use resume copy policy to these files:
(1) Upload file to oss: ossutil will split the big file to many parts, use multipart upload. If
upload is failed, ossutil will record failure information in .ossutil_checkpoint directory
in local file system. When retry, ossutil will read the checkpoint information and resume
upload, if the upload is succeed, ossutil will remove the .ossutil_checkpoint directory.
(2) Download object from oss: ossutil will split the big file to many parts, range get each part.
If download is failed, ossutil wll record failure information in .ossutil_checkpoint directory
in local file system. If success, ossutil will remove the directory.
(3) Copy between oss: ossutil will split the big file to many parts, use Upload Part Copy, and
record failure information in .ossutil_checkpoint directory in local file system. If success,
ossutil will remove the directory.
Warning:
1) Resume copy will not be implemented on small file, if failure happens, ossutil will copy the
whole file the next time.
2) In operation (1) and (3), if failure happens, uploadId that has not been completed may appear in
oss. If the operation success after retry, these uploadId will be completed automatically.
3) When upload file to oss, if .ossutil_checkpoint directory is included in file_url, .ossutil_checkpoint
will not be uploaded to oss. The path of checkpoint directory can be specified by --checkpoint-dir
option, please make sure the directory you specified can be removed.
4) If you remove the uncompleted multipart upload tasks by rm command, may cause resume upload/download/copy
fail the next time(Error: NoSuchUpload). If you want to reupload/download/copy the entire file again,
please remove the checkpoint file in checkpoint directory.
Performance Tuning:
--jobs option or --parallel option (Concurrency Control)
--jobs option controls the amount of concurrency tasks between multi-files, --parallel option controls
the amount of concurrency tasks when work with a file. In default situation, ossutil will calculate the
parallel num according to file size(the option is useless to small file, the file size to use multipart
upload can be specified by --bigfile-threshold option). When batch upload/download/copy files, the total
concurrency tasks num is jobs num multiply by parallel num. The two option can be specified by user, if
the performance of default setting is poor, user can adjust the two options.
Note:
1) If the parallels and jobs number are too big, because of the switching between threads, the performance
of upload/download/copy may decline, so please set the options according to your machine condition. If need
performance tuning, user can set the two options to two small numbers at first and increase them step by step.
2) If the parallels and jobs number are too big, in the case of limited machine resources, error "EOF" may
occur due to the network transfer too slow, in this situation, please reduce the --jobs and --parallel num.
--part-size option
The option specify the part size of resume upload/download/copy of big file. In default situation, ossutil
will calculate the part size and parallel according to file size. When performance tuning or some other needs,
user can set the option.
If the option is specified, part num is: ceil(file size/part size). Note that if --parallel value is bigger
than part num, the extra parallel is feeble, the actual parallel will be part num.
If the part size is too small, it may influence ossutil file upload/download/copy performance, if the part
size is too big, it may influence the actual parallel num, so, please if specify the option, please set it
to a reasonable value.
Batch file migration:
ossutil support batch file migration by transfer files through local file system, the usual usage is:
(1) Batch file upload:
ossutil cp your_dir oss://your_bucket -r -f -u
(2) Batch file download:
ossutil cp oss://your_bucket your_dir -r -f -u
(3) File copy between buckets in the same region:
ossutil cp oss://your_src_bucket oss://your_dest_bucket -r -f -u
The meaning of every option, see help above.
After each migration, please check your report file.
When batch file upload, if the file num is big and no other user modified the corresponding object in
oss during the two uploads, you can use --snapshot-path to accelerate the incremental upload, see more
information in help text of --snapshot-path option above.
The command is:
ossutil cp your_dir oss://your_bucket -r -f -u --shapshot-path=your-path
Usage:
There are three usages:
1) ossutil cp file_url oss://bucket[/prefix] [-r] [-f] [-u] [--output-dir=odir] [--bigfile-threshold=size] [--checkpoint-dir=file] [--snapshot-path=sdir]
The usage upload file in local system to oss. file_url can be file or directory. If file_url
is file, no matter --recursive option is specified or not will not affect the result. If file_url
is directory, even if the directory is empty or only contains one file, we must specify --recursive
option. Mind that, ossutil will upload all sub files and directories(include empty directory) inside
file_url to oss, but differe from shell cp, the first level directory specified by file_url will not