Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Newer
Older
100644 699 lines (601 sloc) 15.855 kb
5cd62ad @dustin Moved HTTP stuff out to a separate file.
dustin authored
1 package main
2
3 import (
b195540 @dustin Automatically gzip proxied view requests.
dustin authored
4 "compress/gzip"
b81598a @dustin When storing content, simultaneously replicate it to another node.
dustin authored
5 "errors"
4f92d95 @dustin Implemented PUT with raw hash value.
dustin authored
6 "fmt"
5cd62ad @dustin Moved HTTP stuff out to a separate file.
dustin authored
7 "io"
b81598a @dustin When storing content, simultaneously replicate it to another node.
dustin authored
8 "io/ioutil"
5cd62ad @dustin Moved HTTP stuff out to a separate file.
dustin authored
9 "log"
33e07e7 @dustin Don't always store every node proxied request.
dustin authored
10 "math/rand"
5cd62ad @dustin Moved HTTP stuff out to a separate file.
dustin authored
11 "net/http"
d55f6a3 @dustin Support storing old file revisions.
dustin authored
12 "strconv"
12fe8b6 @dustin Added ability to list stored oids on a node.
dustin authored
13 "strings"
14 "time"
5cd62ad @dustin Moved HTTP stuff out to a separate file.
dustin authored
15 )
16
78cb430 @dustin Adjusted raw URLs to be in a more consistent space.
dustin authored
17 const (
46a7557 @dustin REST Api for config management.
dustin authored
18 blobPrefix = "/.cbfs/blob/"
19 nodePrefix = "/.cbfs/nodes/"
20 metaPrefix = "/.cbfs/meta/"
21 proxyPrefix = "/.cbfs/viewproxy/"
22 fetchPrefix = "/.cbfs/fetch/"
23 listPrefix = "/.cbfs/list/"
24 configPrefix = "/.cbfs/config/"
dbfed5e @dustin Zip file streaming.
dustin authored
25 zipPrefix = "/.cbfs/zip/"
afaf944 @dustin Added a tar target for streaming out files in bulk.
dustin authored
26 tarPrefix = "/.cbfs/tar/"
0ca96e9 @dustin Added some rudimentary fsck URL.
dustin authored
27 fsckPrefix = "/.cbfs/fsck/"
78cb430 @dustin Adjusted raw URLs to be in a more consistent space.
dustin authored
28 )
29
b81598a @dustin When storing content, simultaneously replicate it to another node.
dustin authored
30 type storInfo struct {
31 node string
32 hs string
33 err error
34 }
35
36 // Given a Reader, we produce a new reader that will duplicate the
37 // stream into the next available node and reproduce that content into
38 // another node. Iff that node successfully stores the content, we
39 // return the hash it computed.
40 //
41 // The returned Reader must be consumed until the input EOFs or is
42 // closed. The returned channel may yield a storInfo struct before
43 // it's closed. If it's closed without yielding a storInfo, there are
44 // no remote nodes available.
12c9396 @dustin Be more consistent about not choosing full nodes.
dustin authored
45 func altStoreFile(r io.Reader, length uint64) (io.Reader, <-chan storInfo) {
b81598a @dustin When storing content, simultaneously replicate it to another node.
dustin authored
46 bgch := make(chan storInfo, 1)
47
f9bfb81 @dustin Use common node finding functions.
dustin authored
48 nodes, err := findRemoteNodes()
12c9396 @dustin Be more consistent about not choosing full nodes.
dustin authored
49 nodes = nodes.withAtLeast(length)
f9bfb81 @dustin Use common node finding functions.
dustin authored
50 if err == nil && len(nodes) > 0 {
b81598a @dustin When storing content, simultaneously replicate it to another node.
dustin authored
51 r1, r2 := newMultiReader(r)
52 r = r2
53
54 go func() {
55 defer close(bgch)
56
57 rv := storInfo{node: nodes[0].Address()}
58
78cb430 @dustin Adjusted raw URLs to be in a more consistent space.
dustin authored
59 rurl := "http://" +
60 nodes[0].Address() + blobPrefix
c7ecf10 @dustin Log the upload using StorageNode's regular %v
dustin authored
61 log.Printf("Piping secondary storage to %v", nodes[0])
6721e0d @dustin Use a longer timeout for the piped connections.
dustin authored
62
78cb430 @dustin Adjusted raw URLs to be in a more consistent space.
dustin authored
63 preq, err := http.NewRequest("POST", rurl, r1)
b81598a @dustin When storing content, simultaneously replicate it to another node.
dustin authored
64 if err != nil {
4be691d @dustin Close with error intentially in specific conditions.
dustin authored
65 r1.CloseWithError(err)
b81598a @dustin When storing content, simultaneously replicate it to another node.
dustin authored
66 rv.err = err
67 bgch <- rv
68 return
69 }
70
6721e0d @dustin Use a longer timeout for the piped connections.
dustin authored
71 client := http.Client{
72 Transport: TimeoutTransport(time.Hour),
73 }
74
75 presp, err := client.Do(preq)
b81598a @dustin When storing content, simultaneously replicate it to another node.
dustin authored
76 if err == nil {
b5025e6 @dustin Change some 204s to 201s.
dustin authored
77 if presp.StatusCode != 201 {
b81598a @dustin When storing content, simultaneously replicate it to another node.
dustin authored
78 rv.err = errors.New(presp.Status)
4be691d @dustin Close with error intentially in specific conditions.
dustin authored
79 r1.CloseWithError(rv.err)
b81598a @dustin When storing content, simultaneously replicate it to another node.
dustin authored
80 bgch <- rv
81 }
82 _, err := io.Copy(ioutil.Discard, presp.Body)
83 if err == nil {
0b3eec3 @dustin Use X-CBFS-Hash consistently.
dustin authored
84 rv.hs = presp.Header.Get("X-CBFS-Hash")
b81598a @dustin When storing content, simultaneously replicate it to another node.
dustin authored
85 }
86 presp.Body.Close()
87 } else {
88 log.Printf("Error http'n to %v: %v", rurl, err)
89 }
90 rv.err = err
91 bgch <- rv
92 }()
93 } else {
94 close(bgch)
95 }
96
97 return r, bgch
98 }
99
78cb430 @dustin Adjusted raw URLs to be in a more consistent space.
dustin authored
100 func doPostRawBlob(w http.ResponseWriter, req *http.Request) {
101 f, err := NewHashRecord(*root, "")
102 if err != nil {
103 log.Printf("Error writing tmp file: %v", err)
104 w.WriteHeader(500)
105 w.Write([]byte(err.Error()))
106 return
107 }
108 defer f.Close()
109
110 sh, length, err := f.Process(req.Body)
111 if err != nil {
112 log.Printf("Error linking in raw hash: %v", err)
113 w.WriteHeader(500)
114 w.Write([]byte(err.Error()))
115 return
116 }
117
736f8a8 @dustin More conservative GC.
dustin authored
118 err = recordBlobOwnership(sh, length, true)
78cb430 @dustin Adjusted raw URLs to be in a more consistent space.
dustin authored
119 if err != nil {
120 log.Printf("Error recording blob ownership: %v", err)
121 w.WriteHeader(500)
122 fmt.Fprintf(w, "Error recording blob ownership: %v", err)
123 return
124 }
125
0b3eec3 @dustin Use X-CBFS-Hash consistently.
dustin authored
126 w.Header().Set("X-CBFS-Hash", sh)
78cb430 @dustin Adjusted raw URLs to be in a more consistent space.
dustin authored
127
b5025e6 @dustin Change some 204s to 201s.
dustin authored
128 w.WriteHeader(201)
78cb430 @dustin Adjusted raw URLs to be in a more consistent space.
dustin authored
129 }
130
4f92d95 @dustin Implemented PUT with raw hash value.
dustin authored
131 func putUserFile(w http.ResponseWriter, req *http.Request) {
31f80c8 @dustin Disallow // in filenames on upload.
dustin authored
132 if strings.Contains(req.URL.Path, "//") {
133 w.WriteHeader(400)
134 fmt.Fprintf(w, "Too many slashes in the path name: %v",
135 req.URL.Path)
136 return
137 }
138
72b19ab @dustin Optionally validate client-supplied hash on upload.
dustin authored
139 f, err := NewHashRecord(*root, req.Header.Get("X-CBFS-Hash"))
5cd62ad @dustin Moved HTTP stuff out to a separate file.
dustin authored
140 if err != nil {
141 log.Printf("Error writing tmp file: %v", err)
142 w.WriteHeader(500)
143 return
144 }
68fef59 @dustin Make a reusable storage/rename Writer.
dustin authored
145 defer f.Close()
5cd62ad @dustin Moved HTTP stuff out to a separate file.
dustin authored
146
12c9396 @dustin Be more consistent about not choosing full nodes.
dustin authored
147 l := req.ContentLength
148 if l < 1 {
149 // If we don't know, guess about a meg.
150 l = 1024 * 1024
151 }
152 r, bgch := altStoreFile(req.Body, uint64(l))
b81598a @dustin When storing content, simultaneously replicate it to another node.
dustin authored
153
154 h, length, err := f.Process(r)
5cd62ad @dustin Moved HTTP stuff out to a separate file.
dustin authored
155 if err != nil {
68fef59 @dustin Make a reusable storage/rename Writer.
dustin authored
156 log.Printf("Error completing blob write: %v", err)
b81598a @dustin When storing content, simultaneously replicate it to another node.
dustin authored
157 w.WriteHeader(500)
158 fmt.Fprintf(w, "Error completing blob write: %v", err)
159 return
5cd62ad @dustin Moved HTTP stuff out to a separate file.
dustin authored
160 }
161
bfb8c93 @dustin Trim some logs down slightly.
dustin authored
162 log.Printf("Wrote %v -> %v", req.URL.Path, h)
5cd62ad @dustin Moved HTTP stuff out to a separate file.
dustin authored
163
164 fm := fileMeta{
5585716 @dustin Support conditional get, range requests.
dustin authored
165 Headers: req.Header,
166 OID: h,
167 Length: length,
6ac6c38 @dustin Record timestamps as UTC.
dustin authored
168 Modified: time.Now().UTC(),
5cd62ad @dustin Moved HTTP stuff out to a separate file.
dustin authored
169 }
170
736f8a8 @dustin More conservative GC.
dustin authored
171 err = recordBlobOwnership(h, length, true)
5cd62ad @dustin Moved HTTP stuff out to a separate file.
dustin authored
172 if err != nil {
e4d22c0 @dustin Renamed AboutNode to StorageNode.
dustin authored
173 log.Printf("Error storing blob ownership: %v", err)
5cd62ad @dustin Moved HTTP stuff out to a separate file.
dustin authored
174 w.WriteHeader(500)
12fe8b6 @dustin Added ability to list stored oids on a node.
dustin authored
175 fmt.Fprintf(w, "Error recording blob ownership: %v", err)
176 return
177 }
178
b81598a @dustin When storing content, simultaneously replicate it to another node.
dustin authored
179 if si, hasStuff := <-bgch; hasStuff {
180 if si.err != nil || si.hs != h {
181 log.Printf("Error in secondary store to %v: %v",
182 si.node, si.err)
183 w.WriteHeader(500)
184 fmt.Fprintf(w, "Error creating secondary copy: %v\n%v",
185 si.err, si.hs)
186 return
187 }
188 }
189
07d7180 @dustin Configurable default version count.
dustin authored
190 revs := globalConfig.DefaultVersionCount
d55f6a3 @dustin Support storing old file revisions.
dustin authored
191 rheader := req.Header.Get("X-CBFS-KeepRevs")
192 if rheader != "" {
193 i, err := strconv.Atoi(rheader)
194 if err == nil {
195 revs = i
196 }
197 }
198
199 err = storeMeta(resolvePath(req), fm, revs)
12fe8b6 @dustin Added ability to list stored oids on a node.
dustin authored
200 if err != nil {
e4d22c0 @dustin Renamed AboutNode to StorageNode.
dustin authored
201 log.Printf("Error storing file meta: %v", err)
12fe8b6 @dustin Added ability to list stored oids on a node.
dustin authored
202 w.WriteHeader(500)
203 fmt.Fprintf(w, "Error recording blob ownership: %v", err)
5cd62ad @dustin Moved HTTP stuff out to a separate file.
dustin authored
204 return
205 }
206
1870072 @dustin Schedule async replication of new items to catch up to min replicas.
dustin authored
207 if globalConfig.MinReplicas > 2 {
208 // We're below min replica count. Start fixing that
209 // up immediately.
210 go increaseReplicaCount(h, length, globalConfig.MinReplicas-2)
211 }
212
b5025e6 @dustin Change some 204s to 201s.
dustin authored
213 w.WriteHeader(201)
5cd62ad @dustin Moved HTTP stuff out to a separate file.
dustin authored
214 }
215
78cb430 @dustin Adjusted raw URLs to be in a more consistent space.
dustin authored
216 func putRawHash(w http.ResponseWriter, req *http.Request) {
217 inputhash := minusPrefix(req.URL.Path, blobPrefix)
4f92d95 @dustin Implemented PUT with raw hash value.
dustin authored
218
78cb430 @dustin Adjusted raw URLs to be in a more consistent space.
dustin authored
219 if inputhash == "" {
4f92d95 @dustin Implemented PUT with raw hash value.
dustin authored
220 w.WriteHeader(400)
221 w.Write([]byte("No oid specified"))
222 return
223 }
224
68fef59 @dustin Make a reusable storage/rename Writer.
dustin authored
225 f, err := NewHashRecord(*root, inputhash)
4f92d95 @dustin Implemented PUT with raw hash value.
dustin authored
226 if err != nil {
227 log.Printf("Error writing tmp file: %v", err)
228 w.WriteHeader(500)
925525c @dustin Send some error messages in a couple places we weren't.
dustin authored
229 w.Write([]byte(err.Error()))
4f92d95 @dustin Implemented PUT with raw hash value.
dustin authored
230 return
231 }
68fef59 @dustin Make a reusable storage/rename Writer.
dustin authored
232 defer f.Close()
4f92d95 @dustin Implemented PUT with raw hash value.
dustin authored
233
b81598a @dustin When storing content, simultaneously replicate it to another node.
dustin authored
234 sh, length, err := f.Process(req.Body)
4f92d95 @dustin Implemented PUT with raw hash value.
dustin authored
235 if err != nil {
68fef59 @dustin Make a reusable storage/rename Writer.
dustin authored
236 log.Printf("Error linking in raw hash: %v", err)
4f92d95 @dustin Implemented PUT with raw hash value.
dustin authored
237 w.WriteHeader(500)
68fef59 @dustin Make a reusable storage/rename Writer.
dustin authored
238 w.Write([]byte(err.Error()))
4f92d95 @dustin Implemented PUT with raw hash value.
dustin authored
239 return
240 }
241
736f8a8 @dustin More conservative GC.
dustin authored
242 err = recordBlobOwnership(inputhash, length, true)
12fe8b6 @dustin Added ability to list stored oids on a node.
dustin authored
243 if err != nil {
244 log.Printf("Error recording blob ownership: %v", err)
245 w.WriteHeader(500)
246 fmt.Fprintf(w, "Error recording blob ownership: %v", err)
247 return
248 }
249
0b3eec3 @dustin Use X-CBFS-Hash consistently.
dustin authored
250 w.Header().Set("X-CBFS-Hash", sh)
b81598a @dustin When storing content, simultaneously replicate it to another node.
dustin authored
251
b5025e6 @dustin Change some 204s to 201s.
dustin authored
252 w.WriteHeader(201)
4f92d95 @dustin Implemented PUT with raw hash value.
dustin authored
253 }
254
255 func doPut(w http.ResponseWriter, req *http.Request) {
b81598a @dustin When storing content, simultaneously replicate it to another node.
dustin authored
256 switch {
46a7557 @dustin REST Api for config management.
dustin authored
257 case req.URL.Path == configPrefix:
258 putConfig(w, req)
78cb430 @dustin Adjusted raw URLs to be in a more consistent space.
dustin authored
259 case strings.HasPrefix(req.URL.Path, blobPrefix):
260 putRawHash(w, req)
cc7221d @dustin File metadata management interface.
dustin authored
261 case strings.HasPrefix(req.URL.Path, metaPrefix):
262 putMeta(w, req, minusPrefix(req.URL.Path, metaPrefix))
14cb4b2 @dustin Block /.cbfs/ from various methods.
dustin authored
263 case strings.HasPrefix(req.URL.Path, "/.cbfs/"):
264 w.WriteHeader(400)
b81598a @dustin When storing content, simultaneously replicate it to another node.
dustin authored
265 default:
4f92d95 @dustin Implemented PUT with raw hash value.
dustin authored
266 putUserFile(w, req)
267 }
268 }
269
5cd62ad @dustin Moved HTTP stuff out to a separate file.
dustin authored
270 func isResponseHeader(s string) bool {
271 switch s {
272 case "Content-Type", "Content-Length":
273 return true
274 }
275 return false
276 }
277
65ad4e7 @dustin GET, PUT and DELETE / should all resolve to defaultPath.
dustin authored
278 func resolvePath(req *http.Request) string {
c339231 @dustin A request to / with no control args will fetch another object.
dustin authored
279 path := req.URL.Path
c010e9a @dustin Get rid of old defaultPath stuff, /blah/ = /blah/index.html
dustin authored
280 // Ignore /, but remove leading / from /blah
b69439e @dustin Fix path resolve so that / properly resolves to index.html
dustin authored
281 for len(path) > 0 && path[0] == '/' {
c010e9a @dustin Get rid of old defaultPath stuff, /blah/ = /blah/index.html
dustin authored
282 path = path[1:]
c339231 @dustin A request to / with no control args will fetch another object.
dustin authored
283 }
284
c010e9a @dustin Get rid of old defaultPath stuff, /blah/ = /blah/index.html
dustin authored
285 if len(path) > 0 && path[len(path)-1] == '/' {
286 path = path + "index.html"
b69439e @dustin Fix path resolve so that / properly resolves to index.html
dustin authored
287 } else if len(path) == 0 {
288 path = "index.html"
c339231 @dustin A request to / with no control args will fetch another object.
dustin authored
289 }
c010e9a @dustin Get rid of old defaultPath stuff, /blah/ = /blah/index.html
dustin authored
290
65ad4e7 @dustin GET, PUT and DELETE / should all resolve to defaultPath.
dustin authored
291 return path
292 }
c339231 @dustin A request to / with no control args will fetch another object.
dustin authored
293
607ce4c @dustin Implement HTTP HEAD.
dustin authored
294 func doHead(w http.ResponseWriter, req *http.Request) {
295 path := resolvePath(req)
296 got := fileMeta{}
297 err := couchbase.Get(path, &got)
298 if err != nil {
299 log.Printf("Error getting file %#v: %v", path, err)
300 w.WriteHeader(404)
301 w.Write([]byte(err.Error()))
302 return
303 }
304
c23323c @dustin Support fetching older revs.
dustin authored
305 if req.FormValue("rev") != "" {
306 w.WriteHeader(400)
307 return
308 }
309
607ce4c @dustin Implement HTTP HEAD.
dustin authored
310 for k, v := range got.Headers {
311 if isResponseHeader(k) {
312 w.Header()[k] = v
313 }
314 }
c23323c @dustin Support fetching older revs.
dustin authored
315
316 oldestRev := got.Revno
317 if len(got.Previous) > 0 {
318 oldestRev = got.Previous[0].Revno
319 }
320
321 w.Header().Set("X-CBFS-Revno", strconv.Itoa(got.Revno))
322 w.Header().Set("X-CBFS-OldestRev", strconv.Itoa(oldestRev))
607ce4c @dustin Implement HTTP HEAD.
dustin authored
323 w.Header().Set("Last-Modified",
324 got.Modified.UTC().Format(http.TimeFormat))
dfeb9d3 @dustin Conditional GET with ETags
dustin authored
325 w.Header().Set("Etag", `"`+got.OID+`"`)
fd2cd29 @dustin Return the Content-Length in HEAD
dustin authored
326 w.Header().Set("Content-Length", fmt.Sprintf("%d", got.Length))
607ce4c @dustin Implement HTTP HEAD.
dustin authored
327
328 w.WriteHeader(200)
329 }
330
d9b75bc @dustin Automatically compress stuff we're serving.
dustin authored
331 type geezyWriter struct {
332 orig http.ResponseWriter
333 w io.Writer
334 }
335
336 func (g *geezyWriter) Header() http.Header {
337 return g.orig.Header()
338 }
339
340 func (g *geezyWriter) Write(data []byte) (int, error) {
341 return g.w.Write(data)
342 }
343
344 func (g *geezyWriter) WriteHeader(status int) {
345 g.orig.WriteHeader(status)
346 }
347
fe1c213 @dustin Only compress specific data types.
dustin authored
348 func shouldGzip(f fileMeta) bool {
349 ct := f.Headers.Get("Content-Type")
350 switch {
351 case strings.HasPrefix(ct, "text/"),
352 strings.HasPrefix(ct, "application/json"):
353 return true
354 }
355 return false
356 }
357
65ad4e7 @dustin GET, PUT and DELETE / should all resolve to defaultPath.
dustin authored
358 func doGetUserDoc(w http.ResponseWriter, req *http.Request) {
359 path := resolvePath(req)
5cd62ad @dustin Moved HTTP stuff out to a separate file.
dustin authored
360 got := fileMeta{}
c339231 @dustin A request to / with no control args will fetch another object.
dustin authored
361 err := couchbase.Get(path, &got)
5cd62ad @dustin Moved HTTP stuff out to a separate file.
dustin authored
362 if err != nil {
c339231 @dustin A request to / with no control args will fetch another object.
dustin authored
363 log.Printf("Error getting file %#v: %v", path, err)
5cd62ad @dustin Moved HTTP stuff out to a separate file.
dustin authored
364 w.WriteHeader(404)
925525c @dustin Send some error messages in a couple places we weren't.
dustin authored
365 w.Write([]byte(err.Error()))
5cd62ad @dustin Moved HTTP stuff out to a separate file.
dustin authored
366 return
367 }
368
c23323c @dustin Support fetching older revs.
dustin authored
369 oid := got.OID
370 respHeaders := got.Headers
371 modified := got.Modified
372 revno := got.Revno
373 oldestRev := revno
374
375 if len(got.Previous) > 0 {
376 oldestRev = got.Previous[0].Revno
377 }
378
379 revnoStr := req.FormValue("rev")
380 if revnoStr != "" {
381 i, err := strconv.Atoi(revnoStr)
382 if err != nil {
383 w.WriteHeader(400)
384 fmt.Fprintf(w, "Invalid revno")
385 return
386 }
387 revno = i
388
389 oid = ""
390 for _, rev := range got.Previous {
391 if rev.Revno == revno {
392 oid = rev.OID
393 modified = rev.Modified
394 respHeaders = rev.Headers
395 break
396 }
397 }
398 if oid == "" {
399 w.WriteHeader(410)
400 fmt.Fprintf(w, "Don't have this file with rev %v", revno)
401 return
402 }
403 }
404
fe1c213 @dustin Only compress specific data types.
dustin authored
405 if canGzip(req) && shouldGzip(got) {
d9b75bc @dustin Automatically compress stuff we're serving.
dustin authored
406 w.Header().Set("Content-Encoding", "gzip")
407 gz := gzip.NewWriter(w)
408 defer gz.Close()
409 w = &geezyWriter{w, gz}
410 }
411
c23323c @dustin Support fetching older revs.
dustin authored
412 w.Header().Set("X-CBFS-Revno", strconv.Itoa(revno))
413 w.Header().Set("X-CBFS-OldestRev", strconv.Itoa(oldestRev))
414
dfeb9d3 @dustin Conditional GET with ETags
dustin authored
415 inm := req.Header.Get("If-None-Match")
416 if len(inm) > 2 {
417 inm = inm[1 : len(inm)-1]
418 if got.OID == inm {
419 w.WriteHeader(304)
420 return
421 }
422 }
423
4d084c5 @dustin Use a common interface for opening a blob.
dustin authored
424 f, err := openBlob(oid)
5cd62ad @dustin Moved HTTP stuff out to a separate file.
dustin authored
425 if err != nil {
509fe96 @dustin Implemented /.cbfs/fetch/ for fetching blob IDs.
dustin authored
426 getBlobFromRemote(w, oid, respHeaders, *cachePercentage)
6f66450 @dustin gofmt
dustin authored
427 return
5cd62ad @dustin Moved HTTP stuff out to a separate file.
dustin authored
428 }
429 defer f.Close()
f579c0e @dustin Support raw oid get.
dustin authored
430
c23323c @dustin Support fetching older revs.
dustin authored
431 for k, v := range respHeaders {
f579c0e @dustin Support raw oid get.
dustin authored
432 if isResponseHeader(k) {
433 w.Header()[k] = v
434 }
435 }
436
c23323c @dustin Support fetching older revs.
dustin authored
437 w.Header().Set("Etag", `"`+oid+`"`)
dfeb9d3 @dustin Conditional GET with ETags
dustin authored
438
c23323c @dustin Support fetching older revs.
dustin authored
439 go recordBlobAccess(oid)
440 http.ServeContent(w, req, path, modified, f)
5cd62ad @dustin Moved HTTP stuff out to a separate file.
dustin authored
441 }
442
4f3e398 @dustin Cleanup references on 404d blob access.
dustin authored
443 func doServeRawBlob(w http.ResponseWriter, req *http.Request, oid string) {
4d084c5 @dustin Use a common interface for opening a blob.
dustin authored
444 f, err := openBlob(oid)
4f3e398 @dustin Cleanup references on 404d blob access.
dustin authored
445 if err != nil {
446 w.WriteHeader(404)
447 fmt.Fprintf(w, "Error opening blob: %v", err)
448 removeBlobOwnershipRecord(oid, serverId)
449 return
450 }
451 defer f.Close()
452
453 w.Header().Set("Content-Type", "application/octet-stream")
454
455 go recordBlobAccess(oid)
456 http.ServeContent(w, req, "", time.Time{}, f)
457 }
458
c23323c @dustin Support fetching older revs.
dustin authored
459 func getBlobFromRemote(w http.ResponseWriter, oid string,
deaf3fe @dustin Minor change to getBlobFromRemote to report errors to copyBlob
dustin authored
460 respHeader http.Header, cachePerc int) error {
1505df5 @mschoch add support for retrieving blobs from remote nodes
mschoch authored
461
462 // Find the owners of this blob
463 ownership := BlobOwnership{}
c23323c @dustin Support fetching older revs.
dustin authored
464 oidkey := "/" + oid
1505df5 @mschoch add support for retrieving blobs from remote nodes
mschoch authored
465 err := couchbase.Get(oidkey, &ownership)
466 if err != nil {
c23323c @dustin Support fetching older revs.
dustin authored
467 log.Printf("Missing ownership record for OID: %v", oid)
1505df5 @mschoch add support for retrieving blobs from remote nodes
mschoch authored
468 // Not sure 404 is the right response here
469 w.WriteHeader(404)
deaf3fe @dustin Minor change to getBlobFromRemote to report errors to copyBlob
dustin authored
470 return err
1505df5 @mschoch add support for retrieving blobs from remote nodes
mschoch authored
471 }
472
5aff851 @dustin Resolve nodes in bulk, returning sorted by recency.
dustin authored
473 nl := ownership.ResolveRemoteNodes()
474
1505df5 @mschoch add support for retrieving blobs from remote nodes
mschoch authored
475 // Loop through the nodes that claim to own this blob
476 // If we encounter any errors along the way, try the next node
5aff851 @dustin Resolve nodes in bulk, returning sorted by recency.
dustin authored
477 for _, sid := range nl {
724d966 @dustin Made StorageNode a Stringer() for more consistent logs.
dustin authored
478 log.Printf("Trying to get %s from %s", oid, sid)
1505df5 @mschoch add support for retrieving blobs from remote nodes
mschoch authored
479
c23323c @dustin Support fetching older revs.
dustin authored
480 resp, err := http.Get(sid.BlobURL(oid))
1505df5 @mschoch add support for retrieving blobs from remote nodes
mschoch authored
481 if err != nil {
4257e06 @dustin Have nodes report their own sizes.
dustin authored
482 log.Printf("Error reading oid %s from node %v",
724d966 @dustin Made StorageNode a Stringer() for more consistent logs.
dustin authored
483 oid, sid)
1505df5 @mschoch add support for retrieving blobs from remote nodes
mschoch authored
484 continue
485 }
7e268df @dustin Always have to close those http response bodies.
dustin authored
486 defer resp.Body.Close()
1505df5 @mschoch add support for retrieving blobs from remote nodes
mschoch authored
487
488 if resp.StatusCode != 200 {
f07a5ee @dustin Logging fix.
dustin authored
489 log.Printf("Error response %v from node %v",
724d966 @dustin Made StorageNode a Stringer() for more consistent logs.
dustin authored
490 resp.Status, sid)
1505df5 @mschoch add support for retrieving blobs from remote nodes
mschoch authored
491 continue
492 }
493
6d38fdb @dustin Send proper headers on proxied data.
dustin authored
494 // Found one, set the headers and send it. Keep a
495 // local copy for good luck.
496
c23323c @dustin Support fetching older revs.
dustin authored
497 for k, v := range respHeader {
6d38fdb @dustin Send proper headers on proxied data.
dustin authored
498 if isResponseHeader(k) {
499 w.Header()[k] = v
500 }
501 }
502 w.WriteHeader(200)
6470130 @dustin Locally store blobs proxied from remote nodes.
dustin authored
503 writeTo := io.Writer(w)
33e07e7 @dustin Don't always store every node proxied request.
dustin authored
504 var hw *hashRecord
505
749795f @dustin Error sooner on remote proxy fetch requests.
dustin authored
506 if cachePerc == 100 || (cachePerc > rand.Intn(100) &&
507 availableSpace() > uint64(ownership.Length)) {
508 hw, err = NewHashRecord(*root, oid)
509 if err == nil {
510 writeTo = io.MultiWriter(hw, w)
33e07e7 @dustin Don't always store every node proxied request.
dustin authored
511 } else {
512 hw = nil
513 }
6470130 @dustin Locally store blobs proxied from remote nodes.
dustin authored
514 }
515
516 length, err := io.Copy(writeTo, resp.Body)
517
68fef59 @dustin Make a reusable storage/rename Writer.
dustin authored
518 if err != nil {
519 log.Printf("Failed to write from remote stream %v", err)
deaf3fe @dustin Minor change to getBlobFromRemote to report errors to copyBlob
dustin authored
520 return err
6470130 @dustin Locally store blobs proxied from remote nodes.
dustin authored
521 } else {
522 // A successful copy with a working hash
523 // record means we should link in and record
524 // our copy of this file.
525 if hw != nil {
526 _, err = hw.Finish()
527 if err == nil {
736f8a8 @dustin More conservative GC.
dustin authored
528 go recordBlobOwnership(oid, length,
529 true)
6470130 @dustin Locally store blobs proxied from remote nodes.
dustin authored
530 }
531 }
1505df5 @mschoch add support for retrieving blobs from remote nodes
mschoch authored
532 }
6470130 @dustin Locally store blobs proxied from remote nodes.
dustin authored
533
5f8150f @dustin Return the error on the local link after a proxy.
dustin authored
534 return err
1505df5 @mschoch add support for retrieving blobs from remote nodes
mschoch authored
535 }
536
537 //if we got to this point, no node in the list actually had it
6d38fdb @dustin Send proper headers on proxied data.
dustin authored
538 log.Printf("Don't have hash file: %v and no remote nodes could help",
c23323c @dustin Support fetching older revs.
dustin authored
539 oid)
1505df5 @mschoch add support for retrieving blobs from remote nodes
mschoch authored
540 w.WriteHeader(500)
c23323c @dustin Support fetching older revs.
dustin authored
541 fmt.Fprintf(w, "Cannot locate blob %v", oid)
deaf3fe @dustin Minor change to getBlobFromRemote to report errors to copyBlob
dustin authored
542 return fmt.Errorf("Can't locate blob %v", oid)
1505df5 @mschoch add support for retrieving blobs from remote nodes
mschoch authored
543 }
544
b195540 @dustin Automatically gzip proxied view requests.
dustin authored
545 func canGzip(req *http.Request) bool {
546 acceptable := req.Header.Get("accept-encoding")
547 return strings.Contains(acceptable, "gzip")
548 }
549
509fe96 @dustin Implemented /.cbfs/fetch/ for fetching blob IDs.
dustin authored
550 type captureResponseWriter struct {
dbfed5e @dustin Zip file streaming.
dustin authored
551 w io.Writer
509fe96 @dustin Implemented /.cbfs/fetch/ for fetching blob IDs.
dustin authored
552 hdr http.Header
553 statusCode int
554 }
555
556 func (c *captureResponseWriter) Header() http.Header {
557 return c.hdr
558 }
559
560 func (c *captureResponseWriter) Write(b []byte) (int, error) {
dbfed5e @dustin Zip file streaming.
dustin authored
561 return c.w.Write(b)
509fe96 @dustin Implemented /.cbfs/fetch/ for fetching blob IDs.
dustin authored
562 }
563
564 func (c *captureResponseWriter) WriteHeader(code int) {
565 c.statusCode = code
566 }
567
568 func doFetchDoc(w http.ResponseWriter, req *http.Request,
569 path string) {
570
749795f @dustin Error sooner on remote proxy fetch requests.
dustin authored
571 ownership := BlobOwnership{}
572 oidkey := "/" + path
573 err := couchbase.Get(oidkey, &ownership)
574 if err != nil {
575 log.Printf("Missing ownership record for OID: %v",
576 path)
577 // Not sure 404 is the right response here
578 w.WriteHeader(404)
579 return
580 }
581
582 if availableSpace() < uint64(ownership.Length) {
fd7167f @dustin Try harder to avoid storing things beyond space limits.
dustin authored
583 w.WriteHeader(500)
584 w.Write([]byte("No free space available."))
585 log.Printf("Someone asked me to get %v, but I'm out of space",
586 path)
587 return
588 }
589
c2c064f @dustin Don't remove dead ownership until something's picked it up.
dustin authored
590 queueBlobFetch(path, req.Header.Get("X-Prevnode"))
f047ac3 @dustin Asynchronously fetch blobs instead of blocking for the update.
dustin authored
591 w.WriteHeader(202)
509fe96 @dustin Implemented /.cbfs/fetch/ for fetching blob IDs.
dustin authored
592 }
593
f579c0e @dustin Support raw oid get.
dustin authored
594 func doGet(w http.ResponseWriter, req *http.Request) {
12fe8b6 @dustin Added ability to list stored oids on a node.
dustin authored
595 switch {
78cb430 @dustin Adjusted raw URLs to be in a more consistent space.
dustin authored
596 case req.URL.Path == blobPrefix:
12fe8b6 @dustin Added ability to list stored oids on a node.
dustin authored
597 doList(w, req)
dbad26f @dustin Put all the /.cbfs/ prefixes at the top.
dustin authored
598 case req.URL.Path == nodePrefix:
1accf60 @dustin Added /.cbfs/nodes/ to get a node list.
dustin authored
599 doListNodes(w, req)
46a7557 @dustin REST Api for config management.
dustin authored
600 case req.URL.Path == configPrefix:
601 doGetConfig(w, req)
509fe96 @dustin Implemented /.cbfs/fetch/ for fetching blob IDs.
dustin authored
602 case strings.HasPrefix(req.URL.Path, fetchPrefix):
603 doFetchDoc(w, req,
604 minusPrefix(req.URL.Path, fetchPrefix))
cc7221d @dustin File metadata management interface.
dustin authored
605 case strings.HasPrefix(req.URL.Path, metaPrefix):
606 doGetMeta(w, req,
607 minusPrefix(req.URL.Path, metaPrefix))
78cb430 @dustin Adjusted raw URLs to be in a more consistent space.
dustin authored
608 case strings.HasPrefix(req.URL.Path, blobPrefix):
4f3e398 @dustin Cleanup references on 404d blob access.
dustin authored
609 doServeRawBlob(w, req, minusPrefix(req.URL.Path, blobPrefix))
56b2526 @dustin View proxying.
dustin authored
610 case *enableViewProxy && strings.HasPrefix(req.URL.Path, proxyPrefix):
611 proxyViewRequest(w, req, minusPrefix(req.URL.Path, proxyPrefix))
d0da46e @mschoch introduced new file listing interface
mschoch authored
612 case strings.HasPrefix(req.URL.Path, listPrefix):
613 doListDocs(w, req, minusPrefix(req.URL.Path, listPrefix))
dbfed5e @dustin Zip file streaming.
dustin authored
614 case strings.HasPrefix(req.URL.Path, zipPrefix):
615 doZipDocs(w, req, minusPrefix(req.URL.Path, zipPrefix))
afaf944 @dustin Added a tar target for streaming out files in bulk.
dustin authored
616 case strings.HasPrefix(req.URL.Path, tarPrefix):
617 doTarDocs(w, req, minusPrefix(req.URL.Path, tarPrefix))
0ca96e9 @dustin Added some rudimentary fsck URL.
dustin authored
618 case strings.HasPrefix(req.URL.Path, fsckPrefix):
619 dofsck(w, req, minusPrefix(req.URL.Path, fsckPrefix))
14cb4b2 @dustin Block /.cbfs/ from various methods.
dustin authored
620 case strings.HasPrefix(req.URL.Path, "/.cbfs/"):
621 w.WriteHeader(400)
12fe8b6 @dustin Added ability to list stored oids on a node.
dustin authored
622 default:
f579c0e @dustin Support raw oid get.
dustin authored
623 doGetUserDoc(w, req)
624 }
625 }
626
78cb430 @dustin Adjusted raw URLs to be in a more consistent space.
dustin authored
627 func minusPrefix(s, prefix string) string {
628 return s[len(prefix):]
629 }
630
37b7b7c @dustin DELETE implementation.
dustin authored
631 func doDeleteOID(w http.ResponseWriter, req *http.Request) {
78cb430 @dustin Adjusted raw URLs to be in a more consistent space.
dustin authored
632 oid := minusPrefix(req.URL.Path, blobPrefix)
736f8a8 @dustin More conservative GC.
dustin authored
633
634 ob, err := getBlobOwnership(oid)
635 if err == nil {
636 n, t := ob.mostRecent()
637 if time.Since(t) < time.Hour {
638 log.Printf("%v was referenced within the last hour by %v, ignoring",
639 oid, n)
640 w.WriteHeader(400)
641 return
642 }
643 }
644 err = removeObject(oid)
37b7b7c @dustin DELETE implementation.
dustin authored
645 if err == nil {
beb8b95 @dustin And DELETE responses should be 204.
dustin authored
646 w.WriteHeader(204)
37b7b7c @dustin DELETE implementation.
dustin authored
647 } else {
648 w.WriteHeader(404)
649 w.Write([]byte(err.Error()))
650 }
651 }
652
653 func doDeleteUserDoc(w http.ResponseWriter, req *http.Request) {
65ad4e7 @dustin GET, PUT and DELETE / should all resolve to defaultPath.
dustin authored
654 err := couchbase.Delete(resolvePath(req))
37b7b7c @dustin DELETE implementation.
dustin authored
655 if err == nil {
beb8b95 @dustin And DELETE responses should be 204.
dustin authored
656 w.WriteHeader(204)
37b7b7c @dustin DELETE implementation.
dustin authored
657 } else {
658 w.WriteHeader(404)
659 w.Write([]byte(err.Error()))
660 }
661 }
662
663 func doDelete(w http.ResponseWriter, req *http.Request) {
664 switch {
78cb430 @dustin Adjusted raw URLs to be in a more consistent space.
dustin authored
665 case strings.HasPrefix(req.URL.Path, blobPrefix):
37b7b7c @dustin DELETE implementation.
dustin authored
666 doDeleteOID(w, req)
14cb4b2 @dustin Block /.cbfs/ from various methods.
dustin authored
667 case strings.HasPrefix(req.URL.Path, "/.cbfs/"):
668 w.WriteHeader(400)
37b7b7c @dustin DELETE implementation.
dustin authored
669 default:
670 doDeleteUserDoc(w, req)
671 }
672 }
673
ca59e9b @dustin Cleaned up the doPost thing.
dustin authored
674 func doPost(w http.ResponseWriter, req *http.Request) {
675 if req.URL.Path == blobPrefix {
676 doPostRawBlob(w, req)
677 } else {
678 w.WriteHeader(http.StatusMethodNotAllowed)
679 }
680 }
681
5cd62ad @dustin Moved HTTP stuff out to a separate file.
dustin authored
682 func httpHandler(w http.ResponseWriter, req *http.Request) {
683 defer req.Body.Close()
684 switch req.Method {
685 case "PUT":
686 doPut(w, req)
78cb430 @dustin Adjusted raw URLs to be in a more consistent space.
dustin authored
687 case "POST":
ca59e9b @dustin Cleaned up the doPost thing.
dustin authored
688 doPost(w, req)
5cd62ad @dustin Moved HTTP stuff out to a separate file.
dustin authored
689 case "GET":
690 doGet(w, req)
607ce4c @dustin Implement HTTP HEAD.
dustin authored
691 case "HEAD":
692 doHead(w, req)
37b7b7c @dustin DELETE implementation.
dustin authored
693 case "DELETE":
694 doDelete(w, req)
5cd62ad @dustin Moved HTTP stuff out to a separate file.
dustin authored
695 default:
696 w.WriteHeader(http.StatusMethodNotAllowed)
697 }
698 }
Something went wrong with that request. Please try again.