mpu concurency

go.mod

@@ -1,10 +1,10 @@
 module s3cli
 
-go 1.20
+go 1.21
 
 require (
-	github.com/aws/aws-sdk-go v1.44.307
-	github.com/johannesboyne/gofakes3 v0.0.0-20230506070712-04da935ef877
+	github.com/aws/aws-sdk-go v1.45.2
+	github.com/johannesboyne/gofakes3 v0.0.0-20230905042043-42c772b34232
 	github.com/spf13/cobra v1.7.0
 )
 

go.sum

@@ -1,6 +1,6 @@
 github.com/aws/aws-sdk-go v1.44.256/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
-github.com/aws/aws-sdk-go v1.44.307 h1:2R0/EPgpZcFSUwZhYImq/srjaOrOfLv5MNRzrFyAM38=
-github.com/aws/aws-sdk-go v1.44.307/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
+github.com/aws/aws-sdk-go v1.45.2 h1:hTong9YUklQKqzrGk3WnKABReb5R8GjbG4Y6dEQfjnk=
+github.com/aws/aws-sdk-go v1.45.2/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -10,8 +10,8 @@ github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9Y
 github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
-github.com/johannesboyne/gofakes3 v0.0.0-20230506070712-04da935ef877 h1:O7syWuYGzre3s73s+NkgB8e0ZvsIVhT/zxNU7V1gHK8=
-github.com/johannesboyne/gofakes3 v0.0.0-20230506070712-04da935ef877/go.mod h1:AxgWC4DDX54O2WDoQO1Ceabtn6IbktjU/7bigor+66g=
+github.com/johannesboyne/gofakes3 v0.0.0-20230905042043-42c772b34232 h1:bLR4m7ytWFI7DNjaaucEs/7cSgLmc4fIXNt0utD+yZ4=
+github.com/johannesboyne/gofakes3 v0.0.0-20230905042043-42c772b34232/go.mod h1:AxgWC4DDX54O2WDoQO1Ceabtn6IbktjU/7bigor+66g=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=

main.go

@@ -1006,6 +1006,7 @@ EnvVar:
 	}
 	rootCmd.AddCommand(mpuCompleteCmd)
 
+	var mpuConcurrency = s3manager.DefaultUploadConcurrency
 	mpuCmd := &cobra.Command{
 		Use:   "mpu <bucket[/key]> [file]",
 		Short: "mpu Object(mpu-create, mpu-upload and mpu-complete)",
@@ -1046,13 +1047,14 @@ EnvVar:
 				key = filepath.Base(args[1])
 			}
 
-			err = sc.mpu(ctx, bucket, key, objectContentType, partSize<<20, fd, metadata)
+			err = sc.mpu(ctx, bucket, key, objectContentType, partSize<<20, mpuConcurrency, fd, metadata)
 
 			return sc.errorHandler(err)
 		},
 	}
 	mpuCmd.Flags().StringVar(&objectContentType, "content-type", "", "Object content-type(auto detect if not specified)")
 	mpuCmd.Flags().Int64("part-size", s3manager.MinUploadPartSize>>20, "MPU part-size in MB")
+	mpuCmd.Flags().IntVar(&mpuConcurrency, "concurrency", mpuConcurrency, "MPU concurrency num")
 	mpuCmd.Flags().StringArrayVar(&objectMetadata, "md", nil, "Object user metadata(format Key:Value)")
 	rootCmd.AddCommand(mpuCmd)
 

s3cli.go

@@ -1703,9 +1703,10 @@ func (sc *S3Cli) mpuComplete(ctx context.Context, bucket, key, uid string, etags
 	return err
 }
 
-func (sc *S3Cli) mpu(ctx context.Context, bucket, key, contentType string, partSize int64, r io.Reader, metadata map[string]*string) error {
+func (sc *S3Cli) mpu(ctx context.Context, bucket, key, contentType string, partSize int64, concurrency int, r io.Reader, metadata map[string]*string) error {
 	uploader := s3manager.NewUploaderWithClient(sc.Client, func(u *s3manager.Uploader) {
 		u.PartSize = partSize
+		u.Concurrency = concurrency
 	})
 
 	mi := &s3manager.UploadInput{