-
Notifications
You must be signed in to change notification settings - Fork 342
/
build.go
141 lines (131 loc) · 4.67 KB
/
build.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
package factory
import (
"context"
"fmt"
"cloud.google.com/go/storage"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/treeverse/lakefs/pkg/block"
"github.com/treeverse/lakefs/pkg/block/azure"
"github.com/treeverse/lakefs/pkg/block/gs"
"github.com/treeverse/lakefs/pkg/block/local"
"github.com/treeverse/lakefs/pkg/block/mem"
"github.com/treeverse/lakefs/pkg/block/params"
s3a "github.com/treeverse/lakefs/pkg/block/s3"
"github.com/treeverse/lakefs/pkg/block/transient"
"github.com/treeverse/lakefs/pkg/logging"
"github.com/treeverse/lakefs/pkg/stats"
"golang.org/x/oauth2/google"
"google.golang.org/api/option"
)
const (
// googleAuthCloudPlatform - Cloud Storage authentication https://cloud.google.com/storage/docs/authentication
googleAuthCloudPlatform = "https://www.googleapis.com/auth/cloud-platform"
)
func BuildBlockAdapter(ctx context.Context, statsCollector stats.Collector, c params.AdapterConfig) (block.Adapter, error) {
blockstore := c.BlockstoreType()
logging.FromContext(ctx).
WithField("type", blockstore).
Info("initialize blockstore adapter")
switch blockstore {
case block.BlockstoreTypeLocal:
p, err := c.BlockstoreLocalParams()
if err != nil {
return nil, err
}
return buildLocalAdapter(ctx, p)
case block.BlockstoreTypeS3:
p, err := c.BlockstoreS3Params()
if err != nil {
return nil, err
}
return buildS3Adapter(ctx, statsCollector, p)
case block.BlockstoreTypeMem, "memory":
return mem.New(ctx), nil
case block.BlockstoreTypeTransient:
return transient.New(ctx), nil
case block.BlockstoreTypeGS:
p, err := c.BlockstoreGSParams()
if err != nil {
return nil, err
}
return buildGSAdapter(ctx, p)
case block.BlockstoreTypeAzure:
p, err := c.BlockstoreAzureParams()
if err != nil {
return nil, err
}
return azure.NewAdapter(ctx, p)
default:
return nil, fmt.Errorf("%w '%s' please choose one of %s",
block.ErrInvalidAddress, blockstore, []string{block.BlockstoreTypeLocal, block.BlockstoreTypeS3, block.BlockstoreTypeAzure, block.BlockstoreTypeMem, block.BlockstoreTypeTransient, block.BlockstoreTypeGS})
}
}
func buildLocalAdapter(ctx context.Context, params params.Local) (*local.Adapter, error) {
adapter, err := local.NewAdapter(params.Path,
local.WithAllowedExternalPrefixes(params.AllowedExternalPrefixes),
local.WithImportEnabled(params.ImportEnabled),
)
if err != nil {
return nil, fmt.Errorf("got error opening a local block adapter with path %s: %w", params.Path, err)
}
logging.FromContext(ctx).WithFields(logging.Fields{
"type": "local",
"path": params.Path,
}).Info("initialized blockstore adapter")
return adapter, nil
}
func BuildS3Client(ctx context.Context, params params.S3) (*s3.Client, error) {
cfg, err := s3a.LoadConfig(ctx, params)
if err != nil {
return nil, err
}
client := s3.NewFromConfig(cfg, s3a.WithClientParams(params))
return client, nil
}
func buildS3Adapter(ctx context.Context, statsCollector stats.Collector, params params.S3) (*s3a.Adapter, error) {
opts := []s3a.AdapterOption{
s3a.WithStatsCollector(statsCollector),
s3a.WithDiscoverBucketRegion(params.DiscoverBucketRegion),
s3a.WithPreSignedExpiry(params.PreSignedExpiry),
s3a.WithDisablePreSigned(params.DisablePreSigned),
s3a.WithDisablePreSignedUI(params.DisablePreSignedUI),
}
if params.ServerSideEncryption != "" {
opts = append(opts, s3a.WithServerSideEncryption(params.ServerSideEncryption))
}
if params.ServerSideEncryptionKmsKeyID != "" {
opts = append(opts, s3a.WithServerSideEncryptionKmsKeyID(params.ServerSideEncryptionKmsKeyID))
}
adapter, err := s3a.NewAdapter(ctx, params, opts...)
if err != nil {
return nil, err
}
logging.FromContext(ctx).WithField("type", "s3").Info("initialized blockstore adapter")
return adapter, nil
}
func BuildGSClient(ctx context.Context, params params.GS) (*storage.Client, error) {
var opts []option.ClientOption
if params.CredentialsFile != "" {
opts = append(opts, option.WithCredentialsFile(params.CredentialsFile))
} else if params.CredentialsJSON != "" {
cred, err := google.CredentialsFromJSON(ctx, []byte(params.CredentialsJSON), googleAuthCloudPlatform)
if err != nil {
return nil, err
}
opts = append(opts, option.WithCredentials(cred))
}
return storage.NewClient(ctx, opts...)
}
func buildGSAdapter(ctx context.Context, params params.GS) (*gs.Adapter, error) {
client, err := BuildGSClient(ctx, params)
if err != nil {
return nil, err
}
adapter := gs.NewAdapter(client,
gs.WithPreSignedExpiry(params.PreSignedExpiry),
gs.WithDisablePreSigned(params.DisablePreSigned),
gs.WithDisablePreSignedUI(params.DisablePreSignedUI),
)
logging.FromContext(ctx).WithField("type", "gs").Info("initialized blockstore adapter")
return adapter, nil
}