-
Notifications
You must be signed in to change notification settings - Fork 785
/
buckets.go
142 lines (129 loc) · 4.5 KB
/
buckets.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
package buckets
import (
"context"
"fmt"
"io/ioutil"
"net/http"
"net/url"
"strings"
"time"
jenkinsv1 "github.com/jenkins-x/jx-api/pkg/apis/jenkins.io/v1"
"github.com/jenkins-x/jx/v2/pkg/cloud"
"github.com/jenkins-x/jx/v2/pkg/util"
"github.com/pkg/errors"
"gocloud.dev/blob"
)
// CreateBucketURL creates a go-cloud URL to a bucket
func CreateBucketURL(name string, kind string, settings *jenkinsv1.TeamSettings) (string, error) {
if kind == "" {
provider := settings.KubeProvider
if provider == "" {
return "", fmt.Errorf("No bucket kind provided nor is a kubernetes provider configured for this team so it could not be defaulted")
}
kind = KubeProviderToBucketScheme(provider)
if kind == "" {
return "", fmt.Errorf("No bucket kind is associated with kubernetes provider %s", provider)
}
}
return kind + "://" + name, nil
}
// KubeProviderToBucketScheme returns the bucket scheme for the cloud provider
func KubeProviderToBucketScheme(provider string) string {
switch provider {
case cloud.AKS:
return "azblob"
case cloud.AWS, cloud.EKS:
return "s3"
case cloud.GKE:
return "gs"
default:
return ""
}
}
// ReadURL reads the given URL from either a http/https endpoint or a bucket URL path.
// if specified the httpFn is a function which can append the user/password or token and/or add a header with the token if using a git provider
func ReadURL(urlText string, timeout time.Duration, httpFn func(urlString string) (string, func(*http.Request), error)) ([]byte, error) {
u, err := url.Parse(urlText)
if err != nil {
return nil, errors.Wrapf(err, "failed to parse URL %s", urlText)
}
var headerFunc func(*http.Request)
switch u.Scheme {
case "http", "https":
if httpFn != nil {
urlText, headerFunc, err = httpFn(urlText)
if err != nil {
return nil, err
}
}
return ReadHTTPURL(urlText, headerFunc, timeout)
default:
return ReadBucketURL(u, timeout)
}
}
// ReadHTTPURL reads the HTTP based URL, modifying the headers as needed, and returns the data or returning an error if a 2xx status is not returned
func ReadHTTPURL(u string, headerFunc func(*http.Request), timeout time.Duration) ([]byte, error) {
httpClient := util.GetClientWithTimeout(timeout)
req, err := http.NewRequest("GET", u, nil)
if err != nil {
return nil, err
}
headerFunc(req)
resp, err := httpClient.Do(req)
if err != nil {
return nil, errors.Wrapf(err, "failed to invoke GET on %s", u)
}
stream := resp.Body
defer stream.Close()
data, err := ioutil.ReadAll(stream)
if err != nil {
return nil, errors.Wrapf(err, "failed to GET data from %s", u)
}
if resp.StatusCode >= 400 {
return data, fmt.Errorf("status %s when performing GET on %s", resp.Status, u)
}
return data, err
}
// ReadBucketURL reads the content of a bucket URL of the for 's3://bucketName/foo/bar/whatnot.txt?param=123'
// where any of the query arguments are applied to the underlying Bucket URL and the path is extracted and resolved
// within the bucket
func ReadBucketURL(u *url.URL, timeout time.Duration) ([]byte, error) {
bucketURL, key := SplitBucketURL(u)
ctx, _ := context.WithTimeout(context.Background(), timeout)
bucket, err := blob.Open(ctx, bucketURL)
if err != nil {
return nil, errors.Wrapf(err, "failed to open bucket %s", bucketURL)
}
data, err := bucket.ReadAll(ctx, key)
if err != nil {
return data, errors.Wrapf(err, "failed to read key %s in bucket %s", key, bucketURL)
}
return data, nil
}
// WriteBucketURL writes the data to a bucket URL of the for 's3://bucketName/foo/bar/whatnot.txt?param=123'
// with the given timeout
func WriteBucketURL(u *url.URL, data []byte, timeout time.Duration) error {
bucketURL, key := SplitBucketURL(u)
return WriteBucket(bucketURL, key, data, timeout)
}
// WriteBucket writes the data to a bucket URL and key of the for 's3://bucketName' and key 'foo/bar/whatnot.txt'
// with the given timeout
func WriteBucket(bucketURL string, key string, data []byte, timeout time.Duration) error {
ctx, _ := context.WithTimeout(context.Background(), timeout)
bucket, err := blob.Open(ctx, bucketURL)
if err != nil {
return errors.Wrapf(err, "failed to open bucket %s", bucketURL)
}
err = bucket.WriteAll(ctx, key, data, nil)
if err != nil {
return errors.Wrapf(err, "failed to write key %s in bucket %s", key, bucketURL)
}
return nil
}
// SplitBucketURL splits the full bucket URL into the URL to open the bucket and the file name to refer to
// within the bucket
func SplitBucketURL(u *url.URL) (string, string) {
u2 := *u
u2.Path = ""
return u2.String(), strings.TrimPrefix(u.Path, "/")
}