/
warm-backend-minio.go
131 lines (115 loc) · 3.77 KB
/
warm-backend-minio.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
// Copyright (c) 2015-2022 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package cmd
import (
"context"
"errors"
"fmt"
"io"
"math"
"net/url"
"strings"
"github.com/minio/madmin-go/v3"
minio "github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
)
type warmBackendMinIO struct {
warmBackendS3
}
var _ WarmBackend = (*warmBackendMinIO)(nil)
const (
maxMultipartPutObjectSize = 1024 * 1024 * 1024 * 1024 * 5
maxPartsCount = 10000
maxPartSize = 1024 * 1024 * 1024 * 5
minPartSize = 1024 * 1024 * 128 // chosen by us to be optimal for HDDs
)
// optimalPartInfo - calculate the optimal part info for a given
// object size.
//
// NOTE: Assumption here is that for any object to be uploaded to any S3 compatible
// object storage it will have the following parameters as constants.
//
// maxPartsCount - 10000
// maxMultipartPutObjectSize - 5TiB
func optimalPartSize(objectSize int64) (partSize int64, err error) {
// object size is '-1' set it to 5TiB.
if objectSize == -1 {
objectSize = maxMultipartPutObjectSize
}
// object size is larger than supported maximum.
if objectSize > maxMultipartPutObjectSize {
err = errors.New("entity too large")
return
}
configuredPartSize := minPartSize
// Use floats for part size for all calculations to avoid
// overflows during float64 to int64 conversions.
partSizeFlt := float64(objectSize / maxPartsCount)
partSizeFlt = math.Ceil(partSizeFlt/float64(configuredPartSize)) * float64(configuredPartSize)
// Part size.
partSize = int64(partSizeFlt)
if partSize == 0 {
return minPartSize, nil
}
return partSize, nil
}
func (m *warmBackendMinIO) Put(ctx context.Context, object string, r io.Reader, length int64) (remoteVersionID, error) {
partSize, err := optimalPartSize(length)
if err != nil {
return remoteVersionID(""), err
}
res, err := m.client.PutObject(ctx, m.Bucket, m.getDest(object), r, length, minio.PutObjectOptions{
StorageClass: m.StorageClass,
PartSize: uint64(partSize),
DisableContentSha256: true,
})
return remoteVersionID(res.VersionID), m.ToObjectError(err, object)
}
func newWarmBackendMinIO(conf madmin.TierMinIO, tier string) (*warmBackendMinIO, error) {
// Validation of credentials
if conf.AccessKey == "" || conf.SecretKey == "" {
return nil, errors.New("both access and secret keys are required")
}
if conf.Bucket == "" {
return nil, errors.New("no bucket name was provided")
}
u, err := url.Parse(conf.Endpoint)
if err != nil {
return nil, err
}
creds := credentials.NewStaticV4(conf.AccessKey, conf.SecretKey, "")
opts := &minio.Options{
Creds: creds,
Secure: u.Scheme == "https",
Transport: globalRemoteTargetTransport,
TrailingHeaders: true,
}
client, err := minio.New(u.Host, opts)
if err != nil {
return nil, err
}
client.SetAppInfo(fmt.Sprintf("minio-tier-%s", tier), ReleaseTag)
core := &minio.Core{Client: client}
return &warmBackendMinIO{
warmBackendS3{
client: client,
core: core,
Bucket: conf.Bucket,
Prefix: strings.TrimSuffix(conf.Prefix, slashSeparator),
},
}, nil
}