-
Notifications
You must be signed in to change notification settings - Fork 99
/
append_file.go
301 lines (248 loc) · 8.46 KB
/
append_file.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
package lib
import (
"fmt"
"os"
"strconv"
"strings"
"time"
oss "github.com/aliyun/aliyun-oss-go-sdk/oss"
)
var specChineseAppendFile = SpecText{
synopsisText: "将本地文件内容以append上传方式上传到oss中的appendable object中",
paramText: "local_file_name oss_object [options]",
syntaxText: `
ossutil appendfromfile local_file_name oss://bucket/object [options]
`,
detailHelpText: `
1) 如果object不存在,可以通过--meta设置object的meta信息,比如输入 --meta "x-oss-meta-author:luxun"
可以设置x-oss-meta-author的值为luxun
2) 如果object已经存在,不可以输入--meta信息,因为oss不支持在已经存在的append object上设置meta
用法:
该命令只有一种用法:
1) ossutil appendfromfile local_file_name oss://bucket/object [--meta=meta-value]
将local_file_name内容以append方式上传到可追加的object
如果输入--meta选项,可以设置object的meta信息
`,
sampleText: `
1) append上传文件内容,不设置meta信息
ossutil appendfromfile local_file_name oss://bucket/object
2) append上传文件内容,设置meta信息
ossutil appendfromfile local_file_name oss://bucket/object --meta "x-oss-meta-author:luxun"
3) 以访问者付费模式上传文件内容
ossutil appendfromfile local_file_name oss://bucket/object --payer requester
`,
}
var specEnglishAppendFile = SpecText{
synopsisText: "Upload the contents of the local file to the oss appendable object by append upload mode",
paramText: "local_file_name oss_object [options]",
syntaxText: `
ossutil appendfromfile local_file_name oss://bucket/object [options]
`,
detailHelpText: `
1) If the object does not exist, you can set the meta information of the object with --meta
for example:
inputting --meta "x-oss-meta-author:luxun" can set the value of x-oss-meta-author to luxun
2) If the object already exists, you can't input the --meta option,
oss does not support setting the meta on the exist appendable object.
Usages:
There is only one usage for this command::
1) ossutil appendfromfile local_file_name oss://bucket/object [--meta=meta-value]
Upload the local_file_name content to the object by append mode
If you input the --meta option, you can set the meta value of the object
`,
sampleText: `
1) Uploads file content by append mode without setting meta value
ossutil appendfromfile local_file_name oss://bucket/object
2) Uploads file content by append mode with setting meta value
ossutil appendfromfile local_file_name oss://bucket/object --meta "x-oss-meta-author:luxun"
3) Uploads file content with requester payment mode
ossutil appendfromfile local_file_name oss://bucket/object --payer requester
`,
}
type AppendProgressListener struct {
lastMilliSecond int64
lastSize int64
currSize int64
}
// ProgressChanged handle progress event
func (l *AppendProgressListener) ProgressChanged(event *oss.ProgressEvent) {
if event.EventType == oss.TransferDataEvent || event.EventType == oss.TransferCompletedEvent {
if l.lastMilliSecond == 0 {
l.lastSize = l.currSize
l.currSize = event.ConsumedBytes
l.lastMilliSecond = time.Now().UnixNano() / 1000 / 1000
} else {
now := time.Now()
cost := now.UnixNano()/1000/1000 - l.lastMilliSecond
if cost > 1000 || event.EventType == oss.TransferCompletedEvent {
l.lastSize = l.currSize
l.currSize = event.ConsumedBytes
l.lastMilliSecond = now.UnixNano() / 1000 / 1000
speed := (float64(l.currSize-l.lastSize) / 1024) / (float64(cost) / 1000)
rate := float64(l.currSize) * 100 / float64(event.TotalBytes)
fmt.Printf("\rtotal append %d(%.2f%%) byte,speed is %.2f(KB/s)", event.ConsumedBytes, rate, speed)
}
}
}
}
type appendFileOptionType struct {
bucketName string
objectName string
encodingType string
fileName string
fileSize int64
ossMeta string
}
type AppendFileCommand struct {
command Command
afOption appendFileOptionType
commonOptions []oss.Option
}
var appendFileCommand = AppendFileCommand{
command: Command{
name: "appendfromfile",
nameAlias: []string{"appendfromfile"},
minArgc: 2,
maxArgc: 2,
specChinese: specChineseAppendFile,
specEnglish: specEnglishAppendFile,
group: GroupTypeNormalCommand,
validOptionNames: []string{
OptionConfigFile,
OptionEndpoint,
OptionAccessKeyID,
OptionAccessKeySecret,
OptionSTSToken,
OptionProxyHost,
OptionProxyUser,
OptionProxyPwd,
OptionEncodingType,
OptionMeta,
OptionMaxUpSpeed,
OptionLogLevel,
OptionRequestPayer,
OptionPassword,
OptionMode,
OptionECSRoleName,
OptionTokenTimeout,
OptionRamRoleArn,
OptionRoleSessionName,
OptionReadTimeout,
OptionConnectTimeout,
OptionSTSRegion,
OptionSkipVerifyCert,
OptionUserAgent,
OptionSignVersion,
OptionRegion,
OptionCloudBoxID,
},
},
}
// function for FormatHelper interface
func (afc *AppendFileCommand) formatHelpForWhole() string {
return afc.command.formatHelpForWhole()
}
func (afc *AppendFileCommand) formatIndependHelp() string {
return afc.command.formatIndependHelp()
}
// Init simulate inheritance, and polymorphism
func (afc *AppendFileCommand) Init(args []string, options OptionMapType) error {
return afc.command.Init(args, options, afc)
}
// RunCommand simulate inheritance, and polymorphism
func (afc *AppendFileCommand) RunCommand() error {
afc.afOption.encodingType, _ = GetString(OptionEncodingType, afc.command.options)
afc.afOption.ossMeta, _ = GetString(OptionMeta, afc.command.options)
srcBucketUrL, err := GetCloudUrl(afc.command.args[1], afc.afOption.encodingType)
if err != nil {
return err
}
if srcBucketUrL.object == "" {
return fmt.Errorf("object key is empty")
}
payer, _ := GetString(OptionRequestPayer, afc.command.options)
if payer != "" {
if payer != strings.ToLower(string(oss.Requester)) {
return fmt.Errorf("invalid request payer: %s, please check", payer)
}
afc.commonOptions = append(afc.commonOptions, oss.RequestPayer(oss.PayerType(payer)))
}
afc.afOption.bucketName = srcBucketUrL.bucket
afc.afOption.objectName = srcBucketUrL.object
// check input file
fileName := afc.command.args[0]
stat, err := os.Stat(fileName)
if err != nil {
return err
}
if stat.IsDir() {
return fmt.Errorf("%s is dir", fileName)
}
if stat.Size() > MaxAppendObjectSize {
return fmt.Errorf("locafile:%s is bigger than %d, it is not supported by append", fileName, MaxAppendObjectSize)
}
afc.afOption.fileName = fileName
afc.afOption.fileSize = stat.Size()
// check object exist or not
client, err := afc.command.ossClient(afc.afOption.bucketName)
if err != nil {
return err
}
bucket, err := client.Bucket(afc.afOption.bucketName)
if err != nil {
return err
}
isExist, err := bucket.IsObjectExist(afc.afOption.objectName, afc.commonOptions...)
if err != nil {
return err
}
if isExist && afc.afOption.ossMeta != "" {
return fmt.Errorf("setting meta on existing append object is not supported")
}
position := int64(0)
if isExist {
//get object size
props, err := bucket.GetObjectMeta(afc.afOption.objectName, afc.commonOptions...)
if err != nil {
return err
}
position, err = strconv.ParseInt(props.Get("Content-Length"), 10, 64)
if err != nil {
return err
}
}
err = afc.AppendFromFile(bucket, position)
return err
}
func (afc *AppendFileCommand) AppendFromFile(bucket *oss.Bucket, position int64) error {
file, err := os.OpenFile(afc.afOption.fileName, os.O_RDONLY, 0660)
if err != nil {
return err
}
defer file.Close()
var options []oss.Option
if afc.afOption.ossMeta != "" {
metas, err := afc.command.parseHeaders(afc.afOption.ossMeta, false)
if err != nil {
return err
}
options, err = afc.command.getOSSOptions(headerOptionMap, metas)
if err != nil {
return err
}
}
var listener *AppendProgressListener = &AppendProgressListener{}
options = append(options, oss.Progress(listener))
options = append(options, afc.commonOptions...)
startT := time.Now()
newPosition, err := bucket.AppendObject(afc.afOption.objectName, file, position, options...)
endT := time.Now()
if err != nil {
return err
} else {
cost := endT.UnixNano()/1000/1000 - startT.UnixNano()/1000/1000
speed := float64(afc.afOption.fileSize) / float64(cost)
fmt.Printf("\nlocal file size is %d,the object new size is %d,average speed is %.2f(KB/s)\n\n", afc.afOption.fileSize, newPosition, speed)
return nil
}
}