service/s3/s3manager: Add Upload Buffer Provider

CHANGELOG_PENDING.md

@@ -1,5 +1,10 @@
 ### SDK Features
+* `service/s3/s3manager`: Add Upload Buffer Provider ([#2792](https://github.com/aws/aws-sdk-go/pull/2792))
+  * Adds a new `BufferProvider` member for specifying how part data can be buffered in memory.
+  * Windows platforms will now default to buffering 1MB per part to reduce contention when uploading files.
+  * Non-Windows platforms will continue to employ a non-buffering behavior.
 
 ### SDK Enhancements
+* `awstesting/integration/performance/s3UploadManager`: Extended to support benchmarking and usage of new `BufferProvider` ([#2792](https://github.com/aws/aws-sdk-go/pull/2792))
 
 ### SDK Bugs

awstesting/integration/integration.go

@@ -8,6 +8,7 @@ import (
 	"crypto/rand"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"os"
 
 	"github.com/aws/aws-sdk-go/aws"
@@ -39,6 +40,39 @@ func UniqueID() string {
 	return fmt.Sprintf("%x", uuid)
 }
 
+// CreateFileOfSize will return an *os.File that is of size bytes
+func CreateFileOfSize(dir string, size int64) (*os.File, error) {
+	file, err := ioutil.TempFile(dir, "s3Bench")
+	if err != nil {
+		return nil, err
+	}
+
+	err = file.Truncate(size)
+	if err != nil {
+		file.Close()
+		os.Remove(file.Name())
+		return nil, err
+	}
+
+	return file, nil
+}
+
+// SizeToName returns a human-readable string for the given size bytes
+func SizeToName(size int) string {
+	units := []string{"B", "KB", "MB", "GB"}
+	i := 0
+	for size >= 1024 {
+		size /= 1024
+		i++
+	}
+
+	if i > len(units)-1 {
+		i = len(units) - 1
+	}
+
+	return fmt.Sprintf("%d%s", size, units[i])
+}
+
 // SessionWithDefaultRegion returns a copy of the integration session with the
 // region set if one was not already provided.
 func SessionWithDefaultRegion(region string) *session.Session {

awstesting/integration/performance/s3UploadManager/README.md

@@ -3,8 +3,18 @@
 Uploads a file to a S3 bucket using the SDK's S3 upload manager. Allows passing
 in custom configuration for the HTTP client and SDK's Upload Manager behavior.
 
-## Usage Example:
+## Build
+### Standalone
+```sh
+go build -tags "integration perftest" -o s3UploadPerfGo ./awstesting/integration/performance/s3UploadManager
+```
+### Benchmarking
+```sh
+go test -tags "integration perftest" -c -o s3UploadPerfGo ./awstesting/integration/performance/s3UploadManager
+```
 
+## Usage Example:
+### Standalone
 ```sh
 AWS_REGION=us-west-2 AWS_PROFILE=aws-go-sdk-team-test ./s3UploadPerfGo \
 -bucket aws-sdk-go-data \
@@ -18,3 +28,16 @@ AWS_REGION=us-west-2 AWS_PROFILE=aws-go-sdk-team-test ./s3UploadPerfGo \
 -client.timeout.connect=1s \
 -client.timeout.response-header=1s
 ```
+
+### Benchmarking
+```sh
+AWS_REGION=us-west-2 AWS_PROFILE=aws-go-sdk-team-test ./s3UploadPerfGo \
+-test.bench=. \
+-test.benchmem \
+-test.benchtime 1x \
+-bucket aws-sdk-go-data \
+-client.idle-conns 1000 \
+-client.idle-conns-host 300 \
+-client.timeout.connect=1s \
+-client.timeout.response-header=1s
+```

awstesting/integration/performance/s3UploadManager/config.go

@@ -6,16 +6,19 @@ import (
 	"flag"
 	"fmt"
 	"net/http"
+	"os"
 	"strings"
 	"time"
 
 	"github.com/aws/aws-sdk-go/service/s3/s3manager"
 )
 
 type Config struct {
-	Bucket, Key string
-	Filename    string
-	LogVerbose  bool
+	Bucket     string
+	Filename   string
+	Size       int64
+	TempDir    string
+	LogVerbose bool
 
 	SDK    SDKConfig
 	Client ClientConfig
@@ -24,10 +27,11 @@ type Config struct {
 func (c *Config) SetupFlags(prefix string, flagset *flag.FlagSet) {
 	flagset.StringVar(&c.Bucket, "bucket", "",
 		"The S3 bucket `name` to upload the object to.")
-	flagset.StringVar(&c.Key, "key", "",
-		"The S3 object key `name` to name the uploaded object.")
 	flagset.StringVar(&c.Filename, "file", "",
 		"The `path` of the local file to upload.")
+	flagset.Int64Var(&c.Size, "size", 0,
+		"The S3 object size in bytes to upload")
+	flagset.StringVar(&c.TempDir, "temp", os.TempDir(), "location to create temporary files")
 	flagset.BoolVar(&c.LogVerbose, "verbose", false,
 		"The output log will include verbose request information")
 
@@ -38,8 +42,8 @@ func (c *Config) SetupFlags(prefix string, flagset *flag.FlagSet) {
 func (c *Config) Validate() error {
 	var errs Errors
 
-	if len(c.Bucket) == 0 || len(c.Key) == 0 || len(c.Filename) == 0 {
-		errs = append(errs, fmt.Errorf("bucket, key, and filename are required"))
+	if len(c.Bucket) == 0 || (c.Size <= 0 && c.Filename == "") {
+		errs = append(errs, fmt.Errorf("bucket and filename/size are required"))
 	}
 
 	if err := c.SDK.Validate(); err != nil {
@@ -60,7 +64,9 @@ type SDKConfig struct {
 	PartSize            int64
 	Concurrency         int
 	WithUnsignedPayload bool
+	WithContentMD5      bool
 	ExpectContinue      bool
+	BufferProvider      s3manager.ReadSeekerWriteToProvider
 }
 
 func (c *SDKConfig) SetupFlags(prefix string, flagset *flag.FlagSet) {
@@ -72,6 +78,8 @@ func (c *SDKConfig) SetupFlags(prefix string, flagset *flag.FlagSet) {
 		"Specifies the number of parts to upload `at once`.")
 	flagset.BoolVar(&c.WithUnsignedPayload, prefix+"unsigned", false,
 		"Specifies if the SDK will use UNSIGNED_PAYLOAD for part SHA256 in request signature.")
+	flagset.BoolVar(&c.WithContentMD5, prefix+"content-md5", true,
+		"Specifies if the SDK should compute the content md5 header for S3 uploads.")
 
 	flagset.BoolVar(&c.ExpectContinue, prefix+"100-continue", true,
 		"Specifies if the SDK requests will wait for the 100 continue response before sending request payload.")