From 62cfac028da9d7a06428305ef8dba031104fc7a8 Mon Sep 17 00:00:00 2001 From: Donovan Kolbly Date: Sat, 4 Mar 2023 17:27:21 -0600 Subject: [PATCH] blob/s3blob: #3214 adds support for per-request s3 Options --- blob/s3blob/s3blob.go | 57 +++++++++++++++++++++++--------------- blob/s3blob/s3blob_test.go | 26 ++++++++++++----- 2 files changed, 54 insertions(+), 29 deletions(-) diff --git a/blob/s3blob/s3blob.go b/blob/s3blob/s3blob.go index b697f2377d..8f7bb2c9d6 100644 --- a/blob/s3blob/s3blob.go +++ b/blob/s3blob/s3blob.go @@ -44,12 +44,12 @@ // s3blob exposes the following types for As: // - Bucket: (V1) *s3.S3; (V2) *s3v2.Client // - Error: (V1) awserr.Error; (V2) any error type returned by the service, notably smithy.APIError -// - ListObject: (V1) s3.Object for objects, s3.CommonPrefix for "directories"; (V2) typesv2.Object for objects, typesv2.CommonPrefix for "directories -// - ListOptions.BeforeList: (V1) *s3.ListObjectsV2Input, or *s3.ListObjectsInput -// when Options.UseLegacyList == true; (V2) *s3v2.ListObjectsV2Input, or *s3v2.ListObjectsInput +// - ListObject: (V1) s3.Object for objects, s3.CommonPrefix for "directories"; (V2) typesv2.Object for objects, typesv2.CommonPrefix for "directories" +// - ListOptions.BeforeList: (V1) *s3.ListObjectsV2Input or *s3.ListObjectsInput +// when Options.UseLegacyList == true; (V2) *s3v2.ListObjectsV2Input or *[]func(*s3v2.Options), or *s3v2.ListObjectsInput // when Options.UseLegacyList == true // - Reader: (V1) s3.GetObjectOutput; (V2) s3v2.GetObjectInput -// - ReaderOptions.BeforeRead: (V1) *s3.GetObjectInput; (V2) *s3v2.GetObjectInput +// - ReaderOptions.BeforeRead: (V1) *s3.GetObjectInput; (V2) *s3v2.GetObjectInput or *[]func(*s3v2.Options) // - Attributes: (V1) s3.HeadObjectOutput; (V2)s3v2.HeadObjectOutput // - CopyOptions.BeforeCopy: *(V1) s3.CopyObjectInput; (V2) s3v2.CopyObjectInput // - WriterOptions.BeforeWrite: (V1) *s3manager.UploadInput, *s3manager.Uploader; (V2) *s3v2.PutObjectInput, *s3v2manager.Uploader @@ -528,20 +528,24 @@ func (b *bucket) ListPaged(ctx context.Context, opts *driver.ListOptions) (*driv func (b *bucket) listObjectsV2(ctx context.Context, in *s3v2.ListObjectsV2Input, opts *driver.ListOptions) (*s3v2.ListObjectsV2Output, error) { if !b.useLegacyList { + var varopt []func(*s3v2.Options) if opts.BeforeList != nil { asFunc := func(i interface{}) bool { - p, ok := i.(**s3v2.ListObjectsV2Input) - if !ok { - return false + if p, ok := i.(**s3v2.ListObjectsV2Input); ok { + *p = in + return true } - *p = in - return true + if p, ok := i.(**[]func(*s3v2.Options)); ok { + *p = &varopt + return true + } + return false } if err := opts.BeforeList(asFunc); err != nil { return nil, err } } - return b.clientV2.ListObjectsV2(ctx, in) + return b.clientV2.ListObjectsV2(ctx, in, varopt...) } // Use the legacy ListObjects request. @@ -589,12 +593,11 @@ func (b *bucket) listObjects(ctx context.Context, in *s3.ListObjectsV2Input, opt if !b.useLegacyList { if opts.BeforeList != nil { asFunc := func(i interface{}) bool { - p, ok := i.(**s3.ListObjectsV2Input) - if !ok { - return false + if p, ok := i.(**s3.ListObjectsV2Input); ok { + *p = in + return true } - *p = in - return true + return false } if err := opts.BeforeList(asFunc); err != nil { return nil, err @@ -776,19 +779,24 @@ func (b *bucket) NewRangeReader(ctx context.Context, key string, offset, length Key: aws.String(key), Range: byteRange, } + var varopt []func(*s3v2.Options) if opts.BeforeRead != nil { asFunc := func(i interface{}) bool { if p, ok := i.(**s3v2.GetObjectInput); ok { *p = in return true } + if p, ok := i.(**[]func(*s3v2.Options)); ok { + *p = &varopt + return true + } return false } if err := opts.BeforeRead(asFunc); err != nil { return nil, err } } - resp, err := b.clientV2.GetObject(ctx, in) + resp, err := b.clientV2.GetObject(ctx, in, varopt...) if err != nil { return nil, err } @@ -960,14 +968,19 @@ func (b *bucket) NewTypedWriter(ctx context.Context, key string, contentType str } if opts.BeforeWrite != nil { asFunc := func(i interface{}) bool { - pu, ok := i.(**s3managerv2.Uploader) - if ok { - *pu = uploaderV2 + // Note that since GCS does not expose AWS's + // Uploader abstraction, there does not appear + // to be any utility in exposing the options + // list to the v2 Uploader's Upload() method. + // Instead, applications can manipulate the + // exposed *Uploader directly, including by + // setting ClientOptions if needed. + if p, ok := i.(**s3managerv2.Uploader); ok { + *p = uploaderV2 return true } - pui, ok := i.(**s3v2.PutObjectInput) - if ok { - *pui = reqV2 + if p, ok := i.(**s3v2.PutObjectInput); ok { + *p = reqV2 return true } return false diff --git a/blob/s3blob/s3blob_test.go b/blob/s3blob/s3blob_test.go index a0686bce20..6c2c5afed4 100644 --- a/blob/s3blob/s3blob_test.go +++ b/blob/s3blob/s3blob_test.go @@ -172,8 +172,11 @@ func (v verifyContentLanguage) ErrorCheck(b *blob.Bucket, err error) error { func (v verifyContentLanguage) BeforeRead(as func(interface{}) bool) error { if v.useV2 { - var req *s3v2.GetObjectInput - if !as(&req) { + var ( + req *s3v2.GetObjectInput + opts *[]func(*s3v2.Options) + ) + if !as(&req) || !as(&opts) { return errors.New("BeforeRead As failed") } return nil @@ -187,8 +190,11 @@ func (v verifyContentLanguage) BeforeRead(as func(interface{}) bool) error { func (v verifyContentLanguage) BeforeWrite(as func(interface{}) bool) error { if v.useV2 { - var req *s3v2.PutObjectInput - if !as(&req) { + var ( + req *s3v2.PutObjectInput + uploader *s3managerv2.Uploader + ) + if !as(&req) || !as(&uploader) { return errors.New("Writer.As failed for PutObjectInput") } req.ContentLanguage = aws.String(language) @@ -233,10 +239,16 @@ func (v verifyContentLanguage) BeforeList(as func(interface{}) bool) error { return errors.New("List.As failed") } } else { - var req *s3v2.ListObjectsV2Input - if !as(&req) { - return errors.New("List.As failed") + var ( + list *s3v2.ListObjectsV2Input + opts *[]func(*s3v2.Options) + ) + // make sure we can extract ALL the underlying GCS types + // for this call + if as(&list) && as(&opts) { + return nil } + return errors.New("List.As failed") } return nil }