Add multipart copy object size threshold configuration option and set…

… it to 128 MB by default
yandex-cloud · Nov 2, 2021 · 82d2e43 · 82d2e43
1 parent 1e49d5c
commit 82d2e43
Show file tree

Hide file tree

Showing 3 changed files with 15 additions and 4 deletions.
diff --git a/api/common/conf_s3.go b/api/common/conf_s3.go
@@ -46,6 +46,8 @@ type S3Config struct {
 	StorageClass string
 	MultipartAge time.Duration
 
+	MultipartCopyThreshold uint64
+
 	UseSSE     bool
 	UseKMS     bool
 	KMSKeyID   string

diff --git a/internal/backend_s3.go b/internal/backend_s3.go
@@ -61,6 +61,9 @@ type S3Backend struct {
 }
 
 func NewS3(bucket string, flags *FlagStorage, config *S3Config) (*S3Backend, error) {
+	if config.MultipartCopyThreshold == 0 {
+		config.MultipartCopyThreshold = 128*1024*1024
+	}
 	awsConfig, err := config.ToAwsConfig(flags)
 	if err != nil {
 		return nil, err
@@ -741,11 +744,9 @@ func (s *S3Backend) CopyBlob(param *CopyBlobInput) (*CopyBlobOutput, error) {
 		metadataDirective = s3.MetadataDirectiveReplace
 	}
 
-	COPY_LIMIT := uint64(5 * 1024 * 1024 * 1024)
-
 	// FIXME Remove additional HEAD query
 
-	if param.Size == nil || param.ETag == nil || (*param.Size > COPY_LIMIT &&
+	if param.Size == nil || param.ETag == nil || (*param.Size > s.config.MultipartCopyThreshold &&
 		(param.Metadata == nil || param.StorageClass == nil)) {
 
 		params := &HeadBlobInput{Key: param.Source}
@@ -772,7 +773,7 @@ func (s *S3Backend) CopyBlob(param *CopyBlobInput) (*CopyBlobOutput, error) {
 
 	from := s.bucket + "/" + param.Source
 
-	if !s.gcs && *param.Size > COPY_LIMIT {
+	if !s.gcs && *param.Size > s.config.MultipartCopyThreshold {
 		reqId, err := s.copyObjectMultipart(int64(*param.Size), from, param.Destination, "", param.ETag, param.Metadata, param.StorageClass)
 		if err != nil {
 			return nil, err

diff --git a/internal/flags.go b/internal/flags.go
@@ -225,6 +225,12 @@ func NewApp() (app *cli.App) {
 			Value: "48h",
 		},
 
+		cli.IntFlag{
+			Name:  "multipart-copy-threshold",
+			Usage: "Threshold for switching from single-part to multipart object copy in MB. Maximum for AWS S3 is 5 GB (default: 128 MB)",
+			Value: 128,
+		},
+
 		/// http://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
 		cli.StringFlag{
 			Name:  "acl",
@@ -601,6 +607,8 @@ func PopulateFlags(c *cli.Context) (ret *FlagStorage) {
 		config.ListV1Ext     = listType == "ext-v1"
 		config.ListV2        = listType == "2"
 
+		config.MultipartCopyThreshold = uint64(c.Int("multipart-copy-threshold")) * 1024 * 1024
+
 		// KMS implies SSE
 		if config.UseKMS {
 			config.UseSSE = true