-
Notifications
You must be signed in to change notification settings - Fork 785
/
buckets.go
109 lines (100 loc) · 3.22 KB
/
buckets.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
package buckets
import (
"context"
"fmt"
jenkinsv1 "github.com/jenkins-x/jx/pkg/apis/jenkins.io/v1"
"github.com/jenkins-x/jx/pkg/util"
"github.com/pkg/errors"
"gocloud.dev/blob"
"io/ioutil"
"net/url"
"strings"
"time"
)
// CreateBucketURL creates a go-cloud URL to a bucket
func CreateBucketURL(name string, kind string, settings *jenkinsv1.TeamSettings) (string, error) {
if kind == "" {
provider := settings.KubeProvider
if provider == "" {
return "", fmt.Errorf("No bucket kind provided nor is a kubernetes provider configured for this team so it could not be defaulted")
}
kind = kubeProviderToBucketKind(provider)
if kind == "" {
return "", fmt.Errorf("No bucket kind is associated with kubernetes provider %s", provider)
}
}
return kind + "://" + name, nil
}
func kubeProviderToBucketKind(provider string) string {
switch provider {
case "gke":
return "gs"
case "aws", "eks":
return "s3"
case "aks", "azure":
return "azblob"
default:
return ""
}
}
// ReadURL reads the given URL from either a http/https endpoint or a bucket URL path.
// if specified the httpFn is a function which can append the user/password or token if using a git provider
func ReadURL(urlText string, timeout time.Duration, httpFn func(urlString string) (string, error)) ([]byte, error) {
u, err := url.Parse(urlText)
if err != nil {
return nil, errors.Wrapf(err, "failed to parse URL %s", urlText)
}
switch u.Scheme {
case "http", "https":
if httpFn != nil {
urlText, err = httpFn(urlText)
if err != nil {
return nil, err
}
}
return ReadHTTPURL(urlText, timeout)
default:
return ReadBucketURL(u, timeout)
}
}
// ReadHTTPURL reads the HTTP based URL and returns the data or returning an error if a 2xx status is not returned
func ReadHTTPURL(u string, timeout time.Duration) ([]byte, error) {
httpClient := util.GetClientWithTimeout(timeout)
resp, err := httpClient.Get(u)
if err != nil {
return nil, errors.Wrapf(err, "failed to invoke GET on %s", u)
}
stream := resp.Body
defer stream.Close()
data, err := ioutil.ReadAll(stream)
if err != nil {
return nil, errors.Wrapf(err, "failed to GET data from %s", u)
}
if resp.StatusCode >= 400 {
return data, fmt.Errorf("status %s when performing GET on %s", resp.Status, u)
}
return data, err
}
// ReadBucketURL reads the content of a bucket URL of the for 's3://bucketName/foo/bar/whatnot.txt?param=123'
// where any of the query arguments are applied to the underlying Bucket URL and the path is extracted and resolved
// within the bucket
func ReadBucketURL(u *url.URL, timeout time.Duration) ([]byte, error) {
bucketURL, key := SplitBucketURL(u)
ctx, _ := context.WithTimeout(context.Background(), timeout)
bucket, err := blob.Open(ctx, bucketURL)
if err != nil {
return nil, errors.Wrapf(err, "failed to open bucket %s", bucketURL)
}
data, err := bucket.ReadAll(ctx, key)
if err != nil {
return data, errors.Wrapf(err, "failed to read key %s in bucket %s", key, bucketURL)
}
return data, nil
}
// SplitBucketURL splits the full bucket URL into the URL to open the bucket and the file name to refer to
// within the bucket
func SplitBucketURL(u *url.URL) (string, string) {
u2 := *u
u2.Path = ""
return u2.String(), strings.TrimPrefix(u.Path, "/")
}