blob/s3blob: google#3214 adds support for per-request s3 Options

blob/s3blob/s3blob.go

@@ -44,12 +44,12 @@
 // s3blob exposes the following types for As:
 //   - Bucket: (V1) *s3.S3; (V2) *s3v2.Client
 //   - Error: (V1) awserr.Error; (V2) any error type returned by the service, notably smithy.APIError
-//   - ListObject: (V1) s3.Object for objects, s3.CommonPrefix for "directories"; (V2) typesv2.Object for objects, typesv2.CommonPrefix for "directories
-//   - ListOptions.BeforeList: (V1) *s3.ListObjectsV2Input, or *s3.ListObjectsInput
-//     when Options.UseLegacyList == true; (V2) *s3v2.ListObjectsV2Input, or *s3v2.ListObjectsInput
+//   - ListObject: (V1) s3.Object for objects, s3.CommonPrefix for "directories"; (V2) typesv2.Object for objects, typesv2.CommonPrefix for "directories"
+//   - ListOptions.BeforeList: (V1) *s3.ListObjectsV2Input or *s3.ListObjectsInput
+//     when Options.UseLegacyList == true; (V2) *s3v2.ListObjectsV2Input or *[]func(*s3v2.Options), or *s3v2.ListObjectsInput
 //     when Options.UseLegacyList == true
 //   - Reader: (V1) s3.GetObjectOutput; (V2) s3v2.GetObjectInput
-//   - ReaderOptions.BeforeRead: (V1) *s3.GetObjectInput; (V2) *s3v2.GetObjectInput
+//   - ReaderOptions.BeforeRead: (V1) *s3.GetObjectInput; (V2) *s3v2.GetObjectInput or *[]func(*s3v2.Options)
 //   - Attributes: (V1) s3.HeadObjectOutput; (V2)s3v2.HeadObjectOutput
 //   - CopyOptions.BeforeCopy: *(V1) s3.CopyObjectInput; (V2) s3v2.CopyObjectInput
 //   - WriterOptions.BeforeWrite: (V1) *s3manager.UploadInput, *s3manager.Uploader; (V2) *s3v2.PutObjectInput, *s3v2manager.Uploader
@@ -528,20 +528,24 @@ func (b *bucket) ListPaged(ctx context.Context, opts *driver.ListOptions) (*driv
 
 func (b *bucket) listObjectsV2(ctx context.Context, in *s3v2.ListObjectsV2Input, opts *driver.ListOptions) (*s3v2.ListObjectsV2Output, error) {
 	if !b.useLegacyList {
+		var varopt []func(*s3v2.Options)
 		if opts.BeforeList != nil {
 			asFunc := func(i interface{}) bool {
-				p, ok := i.(**s3v2.ListObjectsV2Input)
-				if !ok {
-					return false
+				if p, ok := i.(**s3v2.ListObjectsV2Input); ok {
+					*p = in
+					return true
 				}
-				*p = in
-				return true
+				if p, ok := i.(**[]func(*s3v2.Options)); ok {
+					*p = &varopt
+					return true
+				}
+				return false
 			}
 			if err := opts.BeforeList(asFunc); err != nil {
 				return nil, err
 			}
 		}
-		return b.clientV2.ListObjectsV2(ctx, in)
+		return b.clientV2.ListObjectsV2(ctx, in, varopt...)
 	}
 
 	// Use the legacy ListObjects request.
@@ -589,12 +593,11 @@ func (b *bucket) listObjects(ctx context.Context, in *s3.ListObjectsV2Input, opt
 	if !b.useLegacyList {
 		if opts.BeforeList != nil {
 			asFunc := func(i interface{}) bool {
-				p, ok := i.(**s3.ListObjectsV2Input)
-				if !ok {
-					return false
+				if p, ok := i.(**s3.ListObjectsV2Input); ok {
+					*p = in
+					return true
 				}
-				*p = in
-				return true
+				return false
 			}
 			if err := opts.BeforeList(asFunc); err != nil {
 				return nil, err
@@ -776,19 +779,24 @@ func (b *bucket) NewRangeReader(ctx context.Context, key string, offset, length
 			Key:    aws.String(key),
 			Range:  byteRange,
 		}
+		var varopt []func(*s3v2.Options)
 		if opts.BeforeRead != nil {
 			asFunc := func(i interface{}) bool {
 				if p, ok := i.(**s3v2.GetObjectInput); ok {
 					*p = in
 					return true
 				}
+				if p, ok := i.(**[]func(*s3v2.Options)); ok {
+					*p = &varopt
+					return true
+				}
 				return false
 			}
 			if err := opts.BeforeRead(asFunc); err != nil {
 				return nil, err
 			}
 		}
-		resp, err := b.clientV2.GetObject(ctx, in)
+		resp, err := b.clientV2.GetObject(ctx, in, varopt...)
 		if err != nil {
 			return nil, err
 		}
@@ -960,14 +968,19 @@ func (b *bucket) NewTypedWriter(ctx context.Context, key string, contentType str
 		}
 		if opts.BeforeWrite != nil {
 			asFunc := func(i interface{}) bool {
-				pu, ok := i.(**s3managerv2.Uploader)
-				if ok {
-					*pu = uploaderV2
+				// Note that since GCS does not expose AWS's
+				// Uploader abstraction, there does not appear
+				// to be any utility in exposing the options
+				// list to the v2 Uploader's Upload() method.
+				// Instead, applications can manipulate the
+				// exposed *Uploader directly, including by
+				// setting ClientOptions if needed.
+				if p, ok := i.(**s3managerv2.Uploader); ok {
+					*p = uploaderV2
 					return true
 				}
-				pui, ok := i.(**s3v2.PutObjectInput)
-				if ok {
-					*pui = reqV2
+				if p, ok := i.(**s3v2.PutObjectInput); ok {
+					*p = reqV2
 					return true
 				}
 				return false

blob/s3blob/s3blob_test.go

@@ -172,8 +172,11 @@ func (v verifyContentLanguage) ErrorCheck(b *blob.Bucket, err error) error {
 
 func (v verifyContentLanguage) BeforeRead(as func(interface{}) bool) error {
 	if v.useV2 {
-		var req *s3v2.GetObjectInput
-		if !as(&req) {
+		var (
+			req  *s3v2.GetObjectInput
+			opts *[]func(*s3v2.Options)
+		)
+		if !as(&req) || !as(&opts) {
 			return errors.New("BeforeRead As failed")
 		}
 		return nil
@@ -187,8 +190,11 @@ func (v verifyContentLanguage) BeforeRead(as func(interface{}) bool) error {
 
 func (v verifyContentLanguage) BeforeWrite(as func(interface{}) bool) error {
 	if v.useV2 {
-		var req *s3v2.PutObjectInput
-		if !as(&req) {
+		var (
+			req      *s3v2.PutObjectInput
+			uploader *s3managerv2.Uploader
+		)
+		if !as(&req) || !as(&uploader) {
 			return errors.New("Writer.As failed for PutObjectInput")
 		}
 		req.ContentLanguage = aws.String(language)
@@ -233,10 +239,16 @@ func (v verifyContentLanguage) BeforeList(as func(interface{}) bool) error {
 				return errors.New("List.As failed")
 			}
 		} else {
-			var req *s3v2.ListObjectsV2Input
-			if !as(&req) {
-				return errors.New("List.As failed")
+			var (
+				list *s3v2.ListObjectsV2Input
+				opts *[]func(*s3v2.Options)
+			)
+			// make sure we can extract ALL the underlying GCS types
+			// for this call
+			if as(&list) && as(&opts) {
+				return nil
 			}
+			return errors.New("List.As failed")
 		}
 		return nil
 	}