-
Notifications
You must be signed in to change notification settings - Fork 66
/
httpblocksource.go
110 lines (91 loc) · 2.57 KB
/
httpblocksource.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
package blocksources
import (
"bytes"
"errors"
"fmt"
"net/http"
"strings"
)
const MB = 1024 * 1024
var RangedRequestNotSupportedError = errors.New("Ranged request not supported (Server did not respond with 206 Status)")
var ResponseFromServerWasGZiped = errors.New("HTTP response was gzip encoded. Ranges may not match those requested.")
var ClientNoCompression = &http.Client{
Transport: &http.Transport{},
}
func NewHttpBlockSource(
url string,
concurrentRequests int,
resolver BlockSourceOffsetResolver,
verifier BlockVerifier,
) *BlockSourceBase {
return NewBlockSourceBase(
&HttpRequester{
url: url,
client: http.DefaultClient,
},
resolver,
verifier,
concurrentRequests,
4*MB,
)
}
type URLNotFoundError string
func (url URLNotFoundError) Error() string {
return "404 Error on URL: " + string(url)
}
// This class provides the implementation of BlockSourceRequester for BlockSourceBase
// this simplifies creating new BlockSources that satisfy the requirements down to
// writing a request function
type HttpRequester struct {
client *http.Client
url string
}
func (r *HttpRequester) DoRequest(startOffset int64, endOffset int64) (data []byte, err error) {
rangedRequest, err := http.NewRequest("GET", r.url, nil)
if err != nil {
return nil, fmt.Errorf("Error creating request for \"%v\": %v", r.url, err)
}
rangeSpecifier := fmt.Sprintf("bytes=%v-%v", startOffset, endOffset-1)
rangedRequest.ProtoAtLeast(1, 1)
rangedRequest.Header.Add("Range", rangeSpecifier)
rangedRequest.Header.Add("Accept-Encoding", "identity")
rangedResponse, err := r.client.Do(rangedRequest)
if err != nil {
return nil, fmt.Errorf("Error executing request for \"%v\": %v", r.url, err)
}
defer rangedResponse.Body.Close()
if rangedResponse.StatusCode == 404 {
return nil, URLNotFoundError(r.url)
} else if rangedResponse.StatusCode != 206 {
return nil, RangedRequestNotSupportedError
} else if strings.Contains(
rangedResponse.Header.Get("Content-Encoding"),
"gzip",
) {
return nil, ResponseFromServerWasGZiped
} else {
buf := bytes.NewBuffer(make([]byte, 0, endOffset-startOffset))
_, err = buf.ReadFrom(rangedResponse.Body)
if err != nil {
err = fmt.Errorf(
"Failed to read response body for %v (%v-%v): %v",
r.url,
startOffset, endOffset-1,
err,
)
}
data = buf.Bytes()
if int64(len(data)) != endOffset-startOffset {
err = fmt.Errorf(
"Unexpected response length %v (%v): %v",
r.url,
endOffset-startOffset+1,
len(data),
)
}
return
}
}
func (r *HttpRequester) IsFatal(err error) bool {
return true
}