-
Notifications
You must be signed in to change notification settings - Fork 31
/
s3.go
80 lines (74 loc) · 2.42 KB
/
s3.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
package s3
import (
"context"
"encoding/json"
"fmt"
dfv1 "github.com/argoproj-labs/argo-dataflow/api/v1alpha1"
"github.com/argoproj-labs/argo-dataflow/runner/sidecar/sink"
"github.com/aws/aws-sdk-go-v2/aws"
v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
"github.com/aws/aws-sdk-go-v2/service/s3"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"os"
"k8s.io/client-go/kubernetes"
)
type s3Sink struct {
client *s3.Client
bucket string
}
type message struct {
Key string `json:"key"`
Path string `json:"path"`
}
func New(ctx context.Context, kubernetesInterface kubernetes.Interface, namespace string, x dfv1.S3Sink) (sink.Interface, error) {
secretInterface := kubernetesInterface.CoreV1().Secrets(namespace)
var accessKeyID string
{
secretName := x.Credentials.AccessKeyID.Name
secret, err := secretInterface.Get(ctx, secretName, metav1.GetOptions{})
if err != nil {
return nil, fmt.Errorf("failed to get secret %q: %w", secretName, err)
}
accessKeyID = string(secret.Data[x.Credentials.AccessKeyID.Key])
}
var secretAccessKey string
{
secretName := x.Credentials.SecretAccessKey.Name
secret, err := secretInterface.Get(ctx, secretName, metav1.GetOptions{})
if err != nil {
return nil, fmt.Errorf("failed to get secret %q: %w", secretName, err)
}
secretAccessKey = string(secret.Data[x.Credentials.SecretAccessKey.Key])
}
options := s3.Options{
Region: x.Region,
Credentials: aws.CredentialsProviderFunc(func(ctx context.Context) (aws.Credentials, error) {
return aws.Credentials{AccessKeyID: accessKeyID, SecretAccessKey: secretAccessKey}, nil
}),
}
if e := x.Endpoint; e != nil {
options.EndpointResolver = s3.EndpointResolverFunc(func(region string, options s3.EndpointResolverOptions) (aws.Endpoint, error) {
return aws.Endpoint{URL: e.URL, SigningRegion: region, HostnameImmutable: true}, nil
})
}
return s3Sink{client: s3.New(options), bucket: x.Bucket}, nil
}
func (h s3Sink) Sink(msg []byte) error {
m := &message{}
if err := json.Unmarshal(msg, m); err != nil {
return err
}
f, err := os.Open(m.Path)
if err != nil {
return fmt.Errorf("failed to open %q: %w", m.Path, err)
}
_, err = h.client.PutObject(context.Background(), &s3.PutObjectInput{
Bucket: &h.bucket,
Key: &m.Key,
Body: f,
}, s3.WithAPIOptions(
// https://aws.github.io/aws-sdk-go-v2/docs/sdk-utilities/s3/#unseekable-streaming-input
v4.SwapComputePayloadSHA256ForUnsignedPayloadMiddleware,
))
return err
}