s3: add DisableMultipart option

Signed-off-by: Jérôme LOYET <822436+fatpat@users.noreply.github.com>
thanos-io · Feb 13, 2024 · 3ab10f4 · 3ab10f4
1 parent a8d75c5
commit 3ab10f4
Show file tree

Hide file tree

Showing 3 changed files with 42 additions and 19 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -37,10 +37,11 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re
 - [#76](https://github.com/thanos-io/objstore/pull/76) GCS: Query for object names only in `Iter` to possibly improve performance when listing objects.
 - [#85](https://github.com/thanos-io/objstore/pull/85) S3: Allow checksum algorithm to be configured
 - [#92](https://github.com/thanos-io/objstore/pull/92) GCS: Allow using a gRPC client.
-- [#94](https://github.com/thanos-io/objstore/pull/94) Allow timingReadCloser to be seeker 
+- [#94](https://github.com/thanos-io/objstore/pull/94) Allow timingReadCloser to be seeker
 - [#96](https://github.com/thanos-io/objstore/pull/96) Allow nopCloserWithObjectSize to be seeker
 - [#86](https://github.com/thanos-io/objstore/pull/86) GCS: Add HTTP Config to GCS
 - [#99](https://github.com/thanos-io/objstore/pull/99) Swift: Add HTTP_Config
+- [#100](https://github.com/thanos-io/objstore/pull/100) s3: add DisableMultipart option
 
 ### Changed
 - [#38](https://github.com/thanos-io/objstore/pull/38) *: Upgrade minio-go version to `v7.0.45`.

diff --git a/README.md b/README.md
@@ -181,6 +181,7 @@ config:
   list_objects_version: ""
   bucket_lookup_type: auto
   send_content_md5: true
+  disable_multipart: false
   part_size: 67108864
   sse_config:
     type: ""
@@ -354,6 +355,22 @@ config:
   service_account: ""
   use_grpc: false
   grpc_conn_pool_size: 0
+  http_config:
+    idle_conn_timeout: 0s
+    response_header_timeout: 0s
+    insecure_skip_verify: false
+    tls_handshake_timeout: 0s
+    expect_continue_timeout: 0s
+    max_idle_conns: 0
+    max_idle_conns_per_host: 0
+    max_conns_per_host: 0
+    tls_config:
+      ca_file: ""
+      cert_file: ""
+      key_file: ""
+      server_name: ""
+      insecure_skip_verify: false
+    disable_compression: false
 prefix: ""
 ```
 

diff --git a/providers/s3/s3.go b/providers/s3/s3.go
@@ -101,6 +101,7 @@ const (
 var DefaultConfig = Config{
 	PutUserMetadata:  map[string]string{},
 	HTTPConfig:       exthttp.DefaultHTTPConfig,
+	DisableMultipart: false,
 	PartSize:         1024 * 1024 * 64, // 64MB.
 	BucketLookupType: AutoLookup,
 	SendContentMd5:   true, // Default to using MD5.
@@ -128,6 +129,7 @@ type Config struct {
 	ListObjectsVersion string             `yaml:"list_objects_version"`
 	BucketLookupType   BucketLookupType   `yaml:"bucket_lookup_type"`
 	SendContentMd5     bool               `yaml:"send_content_md5"`
+	DisableMultipart   bool               `yaml:"disable_multipart"`
 	// PartSize used for multipart upload. Only used if uploaded object size is known and larger than configured PartSize.
 	// NOTE we need to make sure this number does not produce more parts than 10 000.
 	PartSize    uint64    `yaml:"part_size"`
@@ -150,15 +152,16 @@ type TraceConfig struct {
 
 // Bucket implements the store.Bucket interface against s3-compatible APIs.
 type Bucket struct {
-	logger          log.Logger
-	name            string
-	client          *minio.Client
-	defaultSSE      encrypt.ServerSide
-	putUserMetadata map[string]string
-	storageClass    string
-	partSize        uint64
-	listObjectsV1   bool
-	sendContentMd5  bool
+	logger           log.Logger
+	name             string
+	client           *minio.Client
+	defaultSSE       encrypt.ServerSide
+	putUserMetadata  map[string]string
+	storageClass     string
+	disableMultipart bool
+	partSize         uint64
+	listObjectsV1    bool
+	sendContentMd5   bool
 }
 
 // parseConfig unmarshals a buffer into a Config with default values.
@@ -319,15 +322,16 @@ func NewBucketWithConfig(logger log.Logger, config Config, component string) (*B
 	}
 
 	bkt := &Bucket{
-		logger:          logger,
-		name:            config.Bucket,
-		client:          client,
-		defaultSSE:      sse,
-		putUserMetadata: config.PutUserMetadata,
-		storageClass:    storageClass,
-		partSize:        config.PartSize,
-		listObjectsV1:   config.ListObjectsVersion == "v1",
-		sendContentMd5:  config.SendContentMd5,
+		logger:           logger,
+		name:             config.Bucket,
+		client:           client,
+		defaultSSE:       sse,
+		putUserMetadata:  config.PutUserMetadata,
+		storageClass:     storageClass,
+		disableMultipart: config.DisableMultipart,
+		partSize:         config.PartSize,
+		listObjectsV1:    config.ListObjectsVersion == "v1",
+		sendContentMd5:   config.SendContentMd5,
 	}
 	return bkt, nil
 }
@@ -500,6 +504,7 @@ func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error {
 		r,
 		size,
 		minio.PutObjectOptions{
+			DisableMultipart:     b.disableMultipart,
 			PartSize:             partSize,
 			ServerSideEncryption: sse,
 			UserMetadata:         userMetadata,