/
storage.go
180 lines (158 loc) · 5.06 KB
/
storage.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
/*
Copyright 2015 The Go4 Authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package gcsutil provides tools for accessing Google Cloud Storage until they can be
// completely replaced by cloud.google.com/go/storage.
package gcsutil // import "go4.org/cloud/google/gcsutil"
import (
"encoding/xml"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"os"
"strings"
"cloud.google.com/go/storage"
"go4.org/ctxutil"
"golang.org/x/net/context"
)
const gsAccessURL = "https://storage.googleapis.com"
// An Object holds the name of an object (its bucket and key) within
// Google Cloud Storage.
type Object struct {
Bucket string
Key string
}
func (o *Object) valid() error {
if o == nil {
return errors.New("invalid nil Object")
}
if o.Bucket == "" {
return errors.New("missing required Bucket field in Object")
}
if o.Key == "" {
return errors.New("missing required Key field in Object")
}
return nil
}
// A SizedObject holds the bucket, key, and size of an object.
type SizedObject struct {
Object
Size int64
}
func (o *Object) String() string {
if o == nil {
return "<nil *Object>"
}
return fmt.Sprintf("%v/%v", o.Bucket, o.Key)
}
func (so SizedObject) String() string {
return fmt.Sprintf("%v/%v (%vB)", so.Bucket, so.Key, so.Size)
}
// Makes a simple body-less google storage request
func simpleRequest(method, url_ string) (*http.Request, error) {
req, err := http.NewRequest(method, url_, nil)
if err != nil {
return nil, err
}
req.Header.Set("x-goog-api-version", "2")
return req, err
}
// ErrInvalidRange is used when the server has returned http.StatusRequestedRangeNotSatisfiable.
var ErrInvalidRange = errors.New("gcsutil: requested range not satisfiable")
// GetPartialObject fetches part of a Google Cloud Storage object.
// This function relies on the ctx ctxutil.HTTPClient value being set to an OAuth2
// authorized and authenticated HTTP client.
// If length is negative, the rest of the object is returned.
// It returns ErrInvalidRange if the server replies with http.StatusRequestedRangeNotSatisfiable.
// The caller must call Close on the returned value.
func GetPartialObject(ctx context.Context, obj Object, offset, length int64) (io.ReadCloser, error) {
if offset < 0 {
return nil, errors.New("invalid negative offset")
}
if err := obj.valid(); err != nil {
return nil, err
}
req, err := simpleRequest("GET", gsAccessURL+"/"+obj.Bucket+"/"+obj.Key)
if err != nil {
return nil, err
}
if length >= 0 {
req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", offset, offset+length-1))
} else {
req.Header.Set("Range", fmt.Sprintf("bytes=%d-", offset))
}
req.Cancel = ctx.Done()
res, err := ctxutil.Client(ctx).Do(req)
if err != nil {
return nil, fmt.Errorf("GET (offset=%d, length=%d) failed: %v\n", offset, length, err)
}
if res.StatusCode == http.StatusNotFound {
res.Body.Close()
return nil, os.ErrNotExist
}
if !(res.StatusCode == http.StatusPartialContent || (offset == 0 && res.StatusCode == http.StatusOK)) {
res.Body.Close()
if res.StatusCode == http.StatusRequestedRangeNotSatisfiable {
return nil, ErrInvalidRange
}
return nil, fmt.Errorf("GET (offset=%d, length=%d) got failed status: %v\n", offset, length, res.Status)
}
return res.Body, nil
}
// EnumerateObjects lists the objects in a bucket.
// This function relies on the ctx oauth2.HTTPClient value being set to an OAuth2
// authorized and authenticated HTTP client.
// If after is non-empty, listing will begin with lexically greater object names.
// If limit is non-zero, the length of the list will be limited to that number.
func EnumerateObjects(ctx context.Context, bucket, after string, limit int) ([]*storage.ObjectAttrs, error) {
// Build url, with query params
var params []string
if after != "" {
params = append(params, "marker="+url.QueryEscape(after))
}
if limit > 0 {
params = append(params, fmt.Sprintf("max-keys=%v", limit))
}
query := ""
if len(params) > 0 {
query = "?" + strings.Join(params, "&")
}
req, err := simpleRequest("GET", gsAccessURL+"/"+bucket+"/"+query)
if err != nil {
return nil, err
}
req.Cancel = ctx.Done()
res, err := ctxutil.Client(ctx).Do(req)
if err != nil {
return nil, err
}
defer res.Body.Close()
if res.StatusCode != http.StatusOK {
return nil, fmt.Errorf("gcsutil: bad enumerate response code: %v", res.Status)
}
var xres struct {
Contents []SizedObject
}
if err = xml.NewDecoder(res.Body).Decode(&xres); err != nil {
return nil, err
}
objAttrs := make([]*storage.ObjectAttrs, len(xres.Contents))
for k, o := range xres.Contents {
objAttrs[k] = &storage.ObjectAttrs{
Name: o.Key,
Size: o.Size,
}
}
return objAttrs, nil
}