This repository has been archived by the owner on Sep 15, 2020. It is now read-only.
forked from folbricht/desync
-
Notifications
You must be signed in to change notification settings - Fork 0
/
remotehttp.go
203 lines (182 loc) · 4.63 KB
/
remotehttp.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
package desync
import (
"bytes"
"crypto/tls"
"fmt"
"io"
"io/ioutil"
"net/http"
"net/url"
"path/filepath"
"strings"
"time"
"crypto/x509"
"github.com/pkg/errors"
)
// TrustInsecure determines if invalid certs presented by HTTP stores should
// be accepted.
var TrustInsecure bool
// RemoteHTTP is a remote casync store accessed via HTTP.
type RemoteHTTP struct {
location *url.URL
client *http.Client
errorRetry int
}
// NewRemoteHTTPStore initializes a new store that pulls chunks via HTTP(S) from
// a remote web server. n defines the size of idle connections allowed.
func NewRemoteHTTPStore(location *url.URL, n int, cert string, key string) (*RemoteHTTP, error) {
if location.Scheme != "http" && location.Scheme != "https" {
return nil, fmt.Errorf("unsupported scheme %s, expected http or https", location.Scheme)
}
// Make sure we have a trailing / on the path
u := *location
if !strings.HasSuffix(u.Path, "/") {
u.Path = u.Path + "/"
}
var tr *http.Transport
if cert != "" && key != "" {
// Load client cert
certificate, err := tls.LoadX509KeyPair(cert, key)
if err != nil {
return nil, fmt.Errorf("failed to load client certificate from %s", cert)
}
caCertPool, err := x509.SystemCertPool()
if err != nil {
return nil, fmt.Errorf("failed to create CaCertPool")
}
tr = &http.Transport{
Proxy: http.ProxyFromEnvironment,
DisableCompression: true,
MaxIdleConnsPerHost: n,
IdleConnTimeout: 60 * time.Second,
TLSClientConfig: &tls.Config{
InsecureSkipVerify: TrustInsecure,
Certificates: []tls.Certificate{certificate},
RootCAs: caCertPool,
},
}
} else {
// Build a client with the right size connection pool and optionally disable
// certificate verification.
tr = &http.Transport{
Proxy: http.ProxyFromEnvironment,
DisableCompression: true,
MaxIdleConnsPerHost: n,
IdleConnTimeout: 60 * time.Second,
TLSClientConfig: &tls.Config{InsecureSkipVerify: TrustInsecure},
}
}
client := &http.Client{Transport: tr}
return &RemoteHTTP{location: &u, client: client}, nil
}
// SetTimeout configures the timeout on the HTTP client for all requests
func (r *RemoteHTTP) SetTimeout(timeout time.Duration) {
r.client.Timeout = timeout
}
// SetErrorRetry defines how many HTTP errors are retried. This can be useful
// when dealing with unreliable networks that can timeout or where errors are
// transient.
func (r *RemoteHTTP) SetErrorRetry(n int) {
r.errorRetry = n
}
// GetChunk reads and returns one (compressed!) chunk from the store
func (r *RemoteHTTP) GetChunk(id ChunkID) ([]byte, error) {
sID := id.String()
p := filepath.Join(sID[0:4], sID) + chunkFileExt
u, _ := r.location.Parse(p)
var (
resp *http.Response
err error
attempt int
b []byte
)
for {
attempt++
resp, err = r.client.Get(u.String())
if err != nil {
if attempt >= r.errorRetry {
return nil, errors.Wrap(err, u.String())
}
continue
}
defer resp.Body.Close()
switch resp.StatusCode {
case 200: // expected
case 404:
return nil, ChunkMissing{id}
default:
return nil, fmt.Errorf("unexpected status code %d from %s", resp.StatusCode, p)
}
b, err = ioutil.ReadAll(resp.Body)
if err != nil {
if attempt >= r.errorRetry {
return nil, errors.Wrap(err, u.String())
}
continue
}
break
}
return b, err
}
// HasChunk returns true if the chunk is in the store
func (r *RemoteHTTP) HasChunk(id ChunkID) bool {
sID := id.String()
p := filepath.Join(sID[0:4], sID) + chunkFileExt
u, _ := r.location.Parse(p)
var (
resp *http.Response
err error
attempt int
)
retry:
attempt++
resp, err = r.client.Head(u.String())
if err != nil {
if attempt >= r.errorRetry {
return false
}
goto retry
}
io.Copy(ioutil.Discard, resp.Body)
resp.Body.Close()
switch resp.StatusCode {
case 200:
return true
default:
return false
}
}
// StoreChunk adds a new chunk to the store
func (r *RemoteHTTP) StoreChunk(id ChunkID, b []byte) error {
sID := id.String()
p := filepath.Join(sID[0:4], sID) + chunkFileExt
u, _ := r.location.Parse(p)
var (
resp *http.Response
err error
attempt int
)
retry:
attempt++
req, err := http.NewRequest("PUT", u.String(), bytes.NewReader(b))
if err != nil {
return err
}
resp, err = r.client.Do(req)
if err != nil {
if attempt >= r.errorRetry {
return err
}
goto retry
}
defer resp.Body.Close()
msg, _ := ioutil.ReadAll(resp.Body)
if resp.StatusCode != 200 {
return errors.New(string(msg))
}
return nil
}
func (r *RemoteHTTP) String() string {
return r.location.String()
}
func (s RemoteHTTP) Close() error { return nil }