generated from keboola/template
-
Notifications
You must be signed in to change notification settings - Fork 0
/
storage_file_download.go
128 lines (113 loc) · 3.61 KB
/
storage_file_download.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
package keboola
import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"strings"
"gocloud.dev/blob"
"github.com/keboola/go-client/pkg/keboola/storage_file_upload/abs"
"github.com/keboola/go-client/pkg/keboola/storage_file_upload/gcs"
"github.com/keboola/go-client/pkg/keboola/storage_file_upload/s3"
)
type downloadConfig struct {
transport http.RoundTripper
}
type DownloadOption func(c *downloadConfig)
func WithDownloadTransport(transport http.RoundTripper) DownloadOption {
return func(c *downloadConfig) {
c.transport = transport
}
}
func Download(ctx context.Context, file *FileDownloadCredentials) ([]byte, error) {
if file.IsSliced {
return nil, fmt.Errorf("cannot download a sliced file as a whole file")
}
return DownloadSlice(ctx, file, "")
}
func DownloadManifest(ctx context.Context, file *FileDownloadCredentials) (SlicesList, error) {
rawManifest, err := DownloadSlice(ctx, file, ManifestFileName)
if err != nil {
return nil, fmt.Errorf("cannot download manifest: %w", err)
}
manifest := &SlicedFileManifest{}
err = json.Unmarshal(rawManifest, manifest)
if err != nil {
return nil, fmt.Errorf("cannot map file contents to manifest: %w", err)
}
dstURL, err := file.DestinationURL()
if err != nil {
return nil, err
}
res := SlicesList{}
for _, slice := range manifest.Entries {
if !strings.HasPrefix(slice.URL, dstURL) {
return nil, fmt.Errorf(`slice URL "%s" seems wrong: %w`, slice.URL, err)
}
res = append(res, strings.TrimPrefix(slice.URL, dstURL))
}
return res, nil
}
func DownloadSlice(ctx context.Context, file *FileDownloadCredentials, slice string) (out []byte, err error) {
reader, err := DownloadSliceReader(ctx, file, slice)
if err != nil {
return nil, err
}
out, err = io.ReadAll(reader)
if closeErr := reader.Close(); err == nil && closeErr != nil {
err = closeErr
}
if err != nil {
return nil, err
}
return out, nil
}
func DownloadReader(ctx context.Context, file *FileDownloadCredentials) (io.ReadCloser, error) {
return DownloadSliceReader(ctx, file, "")
}
func DownloadManifestReader(ctx context.Context, file *FileDownloadCredentials) (io.ReadCloser, error) {
return DownloadSliceReader(ctx, file, ManifestFileName)
}
func DownloadSliceReader(ctx context.Context, file *FileDownloadCredentials, slice string, opts ...DownloadOption) (io.ReadCloser, error) {
c := downloadConfig{}
for _, opt := range opts {
opt(&c)
}
switch file.Provider {
case abs.Provider:
return abs.NewDownloadReader(ctx, file.ABSDownloadParams, slice, c.transport)
case gcs.Provider:
return gcs.NewDownloadReader(ctx, file.GCSDownloadParams, slice, c.transport)
case s3.Provider:
return s3.NewDownloadReader(ctx, file.S3DownloadParams, file.Region, slice, c.transport)
default:
return nil, fmt.Errorf(`unsupported provider "%s"`, file.Provider)
}
}
func GetFileAttributes(ctx context.Context, file *FileDownloadCredentials, slice string, opts ...DownloadOption) (*FileAttributes, error) {
c := downloadConfig{}
for _, opt := range opts {
opt(&c)
}
var attrs *blob.Attributes
var err error
switch file.Provider {
case abs.Provider:
attrs, err = abs.GetFileAttributes(ctx, file.ABSDownloadParams, slice, c.transport)
case gcs.Provider:
attrs, err = gcs.GetFileAttributes(ctx, file.GCSDownloadParams, slice, c.transport)
case s3.Provider:
attrs, err = s3.GetFileAttributes(ctx, file.S3DownloadParams, file.Region, slice, c.transport)
default:
return nil, fmt.Errorf(`unsupported provider "%s"`, file.Provider)
}
if err != nil {
return nil, err
}
return &FileAttributes{
ContentType: attrs.ContentType,
ModTime: attrs.ModTime,
Size: attrs.Size,
}, nil
}