Merge pull request #100 from fatpat/addDisableMultipart

s3: add DisableMultipart option

CHANGELOG.md

@@ -45,6 +45,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re
 - [#86](https://github.com/thanos-io/objstore/pull/86) GCS: Add HTTP Config to GCS
 - [#99](https://github.com/thanos-io/objstore/pull/99) Swift: Add HTTP_Config
 - [#108](https://github.com/thanos-io/objstore/pull/108) Metrics: Add native histogram definitions to histograms
+- [#100](https://github.com/thanos-io/objstore/pull/100) s3: add DisableMultipart option
 
 ### Changed
 - [#38](https://github.com/thanos-io/objstore/pull/38) *: Upgrade minio-go version to `v7.0.45`.

README.md

@@ -181,6 +181,7 @@ config:
   list_objects_version: ""
   bucket_lookup_type: auto
   send_content_md5: true
+  disable_multipart: false
   part_size: 67108864
   sse_config:
     type: ""

providers/s3/s3.go

@@ -101,6 +101,7 @@ const (
 var DefaultConfig = Config{
 	PutUserMetadata:  map[string]string{},
 	HTTPConfig:       exthttp.DefaultHTTPConfig,
+	DisableMultipart: false,
 	PartSize:         1024 * 1024 * 64, // 64MB.
 	BucketLookupType: AutoLookup,
 	SendContentMd5:   true, // Default to using MD5.
@@ -128,6 +129,7 @@ type Config struct {
 	ListObjectsVersion string             `yaml:"list_objects_version"`
 	BucketLookupType   BucketLookupType   `yaml:"bucket_lookup_type"`
 	SendContentMd5     bool               `yaml:"send_content_md5"`
+	DisableMultipart   bool               `yaml:"disable_multipart"`
 	// PartSize used for multipart upload. Only used if uploaded object size is known and larger than configured PartSize.
 	// NOTE we need to make sure this number does not produce more parts than 10 000.
 	PartSize    uint64    `yaml:"part_size"`
@@ -150,15 +152,16 @@ type TraceConfig struct {
 
 // Bucket implements the store.Bucket interface against s3-compatible APIs.
 type Bucket struct {
-	logger          log.Logger
-	name            string
-	client          *minio.Client
-	defaultSSE      encrypt.ServerSide
-	putUserMetadata map[string]string
-	storageClass    string
-	partSize        uint64
-	listObjectsV1   bool
-	sendContentMd5  bool
+	logger           log.Logger
+	name             string
+	client           *minio.Client
+	defaultSSE       encrypt.ServerSide
+	putUserMetadata  map[string]string
+	storageClass     string
+	disableMultipart bool
+	partSize         uint64
+	listObjectsV1    bool
+	sendContentMd5   bool
 }
 
 // parseConfig unmarshals a buffer into a Config with default values.
@@ -319,15 +322,16 @@ func NewBucketWithConfig(logger log.Logger, config Config, component string) (*B
 	}
 
 	bkt := &Bucket{
-		logger:          logger,
-		name:            config.Bucket,
-		client:          client,
-		defaultSSE:      sse,
-		putUserMetadata: config.PutUserMetadata,
-		storageClass:    storageClass,
-		partSize:        config.PartSize,
-		listObjectsV1:   config.ListObjectsVersion == "v1",
-		sendContentMd5:  config.SendContentMd5,
+		logger:           logger,
+		name:             config.Bucket,
+		client:           client,
+		defaultSSE:       sse,
+		putUserMetadata:  config.PutUserMetadata,
+		storageClass:     storageClass,
+		disableMultipart: config.DisableMultipart,
+		partSize:         config.PartSize,
+		listObjectsV1:    config.ListObjectsVersion == "v1",
+		sendContentMd5:   config.SendContentMd5,
 	}
 	return bkt, nil
 }
@@ -500,6 +504,7 @@ func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error {
 		r,
 		size,
 		minio.PutObjectOptions{
+			DisableMultipart:     b.disableMultipart,
 			PartSize:             partSize,
 			ServerSideEncryption: sse,
 			UserMetadata:         userMetadata,