/
upload.go
319 lines (301 loc) · 9.17 KB
/
upload.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
package httpx
import (
"bytes"
"errors"
"fmt"
"github.com/hetianyi/gox"
"github.com/hetianyi/gox/cache"
"io"
"net/http"
"reflect"
"regexp"
)
const (
ContentTypePattern = "^multipart/form-data; boundary=(.*)$"
ContentDispositionPattern = "^Content-Disposition: form-data; name=\"([^\"]*)\"(; filename=\"([^\"]*)\".*)?$"
)
var (
newLineMarker = []byte{13, 10}
RegexContentTypePattern = regexp.MustCompile(ContentTypePattern)
RegexContentDispositionPattern = regexp.MustCompile(ContentDispositionPattern)
formReaderType = reflect.TypeOf(&FileFormReader{})
)
// FileTransactionProcessor is a model for processing one single file of a form.
type FileTransactionProcessor struct {
Before func() error
Write func(bs []byte) error
Success func() error
Error func(err error)
}
// FileFormReader processes io operation during file upload.
type FileFormReader struct {
request *http.Request
unReadableBuffer *bytes.Buffer
atomByte []byte
newLineBytesPair []byte
buffer []byte
newLineBuffer *bytes.Buffer
}
// FileUploadHandler defines a handler for processing http file upload.
type FileUploadHandler struct {
Request *http.Request
paraBoundary string
endParaBoundary string
separator []byte
separatorTestBuffer []byte
separatorMergeBuffer []byte
formReader *FileFormReader
// call when read a plain text field.
OnFormField func(paraName, paraValue string)
// call when about begin to read file body from form, need to provide an io.WriteCloser to write file bytes.
OnFileField func(paraName, fileName string) *FileTransactionProcessor
}
// Unread return extra read bytes for next read.
func (reader *FileFormReader) Unread(read []byte) {
reader.unReadableBuffer.Write(read)
}
// Read reads bytes from stream, if the buffer has bytes remain, read it first, then read from form body.
func (reader *FileFormReader) Read(buff []byte) (int, error) {
// if buffer of FileFormReader has bytes cached before(from Unread()), then read it first.
if reader.unReadableBuffer.Len() > 0 {
if len(buff) <= reader.unReadableBuffer.Len() {
return reader.unReadableBuffer.Read(buff)
}
offsetPos, err := reader.unReadableBuffer.Read(buff)
if err != nil {
return 0, err
}
// read directly from reader
len, err := reader.request.Body.Read(buff[offsetPos:])
if err != nil && err != io.EOF {
return 0, err
}
return offsetPos + len, err
}
// read directly from reader
return reader.request.Body.Read(buff)
}
// Parse begin to read request entity and parse form field
func (handler *FileUploadHandler) Parse() error {
defer func() {
handler.formReader.newLineBuffer.Reset()
handler.formReader.unReadableBuffer.Reset()
handler.formReader.request = nil
cache.ReCacheResource(handler.formReader)
handler.formReader = nil
}()
handler.formReader = cache.ApplyResource(formReaderType, func() interface{} {
return &FileFormReader{
request: handler.Request,
unReadableBuffer: new(bytes.Buffer),
atomByte: make([]byte, 1),
newLineBytesPair: make([]byte, 2),
newLineBuffer: new(bytes.Buffer),
buffer: make([]byte, 1024*30),
}
}).(*FileFormReader)
handler.formReader.request = handler.Request
var fileIndex = 0
headerContentType := handler.Request.Header["Content-Type"]
contentType := ""
if headerContentType != nil && len(headerContentType) > 0 {
contentType = headerContentType[0]
}
if RegexContentTypePattern.Match([]byte(contentType)) {
boundary := RegexContentTypePattern.ReplaceAllString(contentType, "${1}")
handler.paraBoundary = "--" + boundary
handler.endParaBoundary = "--" + boundary + "--"
handler.separator = []byte("\r\n" + handler.paraBoundary)
handler.separatorTestBuffer = make([]byte, len(handler.separator))
handler.separatorMergeBuffer = make([]byte, len(handler.separator)*2)
for {
line, err := handler.formReader.readNextLine()
if err != nil {
return err
}
// if it is paraSeparator, then start read new form text field or file field
if handler.paraBoundary == line {
contentDisposition, err := handler.formReader.readNextLine()
if err != nil {
return err
}
mat1 := RegexContentDispositionPattern.Match([]byte(contentDisposition))
paramName := ""
paramValue := ""
if mat1 {
paramName = RegexContentDispositionPattern.ReplaceAllString(contentDisposition, "${1}")
}
paramContentType, err := handler.formReader.readNextLine()
if err != nil {
return err
}
if paramContentType == "" { // read text parameter field
param, err := handler.formReader.readNextLine()
if err != nil {
return err
}
paramValue = param
handler.OnFormField(paramName, paramValue)
} else { // parse content type
mat2 := RegexContentDispositionPattern.Match([]byte(contentDisposition))
fileName := ""
if mat2 {
fileName = RegexContentDispositionPattern.ReplaceAllString(contentDisposition, "${3}")
}
_, err = handler.formReader.readNextLine() // read blank line
if err != nil {
return err
}
// read file body
processor := handler.OnFileField(paramName, fileName)
if processor == nil {
return errors.New("file processor cannot be nil")
}
err := handler.readFileBody(fileName, processor)
if err != nil {
handleError(processor, err)
}
if err != nil {
return err
}
fileIndex++
}
} else if handler.endParaBoundary == line {
// form stream hit end
break
} else {
fmt.Println("unknown line")
}
}
}
return nil
}
// readNextLine reads next form field meta string.
func (reader *FileFormReader) readNextLine() (string, error) {
reader.newLineBuffer.Reset()
for {
len, err := reader.Read(reader.atomByte)
if err != nil && err != io.EOF {
return "", err
}
if len != 1 {
return "", errors.New("error read from stream[0]")
}
reader.newLineBytesPair[0] = reader.newLineBytesPair[1]
reader.newLineBytesPair[1] = reader.atomByte[0]
reader.newLineBuffer.Write(reader.atomByte)
if bytes.Equal(newLineMarker, reader.newLineBytesPair) {
return string(reader.newLineBuffer.Bytes()[0 : reader.newLineBuffer.Len()-2]), nil
}
}
}
// readFileBody reads a file body part.
func (handler *FileUploadHandler) readFileBody(fileName string, processor *FileTransactionProcessor) error {
if processor.Before != nil {
err := processor.Before()
if err != nil {
return err
}
}
separatorLength := len(handler.separator)
for {
len1, err := handler.formReader.Read(handler.formReader.buffer)
if err != nil && err != io.EOF {
return err
}
if len1 == 0 {
return errors.New("read file body failed[0]")
}
// whether buff1 contains separator
pos := bytes.Index(handler.formReader.buffer, handler.separator)
if pos != -1 {
if processor.Write != nil {
err := processor.Write(handler.formReader.buffer[0:pos])
if err != nil {
return err
}
}
handler.formReader.Unread(handler.formReader.buffer[pos+2 : len1]) // skip "\r\n"
if processor.Success != nil {
err := processor.Success()
if err != nil {
return err
}
}
break
} else {
len2, err := handler.formReader.Read(handler.separatorTestBuffer)
if err != nil && err != io.EOF {
return err
}
if len2 == 0 {
return errors.New("read file body failed[1]")
}
// []byte tail is last bytes of buff1 and first bytes of buff2 in case broken separator.
//
if len1 >= separatorLength {
ByteCopy(handler.separatorMergeBuffer, 0, separatorLength, handler.formReader.buffer[len1-separatorLength:len1])
}
if len2 >= separatorLength {
ByteCopy(handler.separatorMergeBuffer, separatorLength, len(handler.separatorMergeBuffer), handler.separatorTestBuffer[0:separatorLength])
}
i2 := bytes.Index(handler.separatorMergeBuffer, handler.separator)
if i2 != -1 {
if i2 < separatorLength {
if processor.Write != nil {
err := processor.Write(handler.formReader.buffer[0 : len1-i2])
if err != nil {
return err
}
}
handler.formReader.Unread(handler.formReader.buffer[len1-i2+2 : len1])
handler.formReader.Unread(handler.separatorTestBuffer[0:len2])
} else {
if processor.Write != nil {
err := processor.Write(handler.formReader.buffer[0:len1])
if err != nil {
return err
}
}
handler.formReader.Unread(handler.separatorTestBuffer[i2-separatorLength+2 : len2])
}
if processor.Success != nil {
err := processor.Success()
if err != nil {
return err
}
}
break
} else {
if processor.Write != nil {
err := processor.Write(handler.formReader.buffer[0:len1])
if err != nil {
return err
}
}
if err != nil {
return err
}
handler.formReader.Unread(handler.separatorTestBuffer[0:len2])
}
}
}
return nil
}
func handleError(processor *FileTransactionProcessor, err error) {
if processor.Error != nil {
gox.Try(func() {
processor.Error(err)
}, func(i interface{}) {})
}
}
// ByteCopy copies bytes
func ByteCopy(src []byte, start int, end int, cp []byte) {
for i := range src {
if i >= start && i < end {
src[i] = cp[i]
} else {
break
}
}
}