diff --git a/.changelog/a7aeef887867446a8c517f39c2be006e.json b/.changelog/a7aeef887867446a8c517f39c2be006e.json new file mode 100644 index 00000000000..e0469ccbe20 --- /dev/null +++ b/.changelog/a7aeef887867446a8c517f39c2be006e.json @@ -0,0 +1,8 @@ +{ + "id": "a7aeef88-7867-446a-8c51-7f39c2be006e", + "type": "feature", + "description": "added custom paginators for listMultipartUploads and ListObjectVersions", + "modules": [ + "service/s3" + ] +} \ No newline at end of file diff --git a/service/s3/handwritten_paginators.go b/service/s3/handwritten_paginators.go new file mode 100644 index 00000000000..3d0b25100d8 --- /dev/null +++ b/service/s3/handwritten_paginators.go @@ -0,0 +1,204 @@ +package s3 + +import ( + "context" + "fmt" +) + +// ListObjectVersionsAPIClient is a client that implements the ListObjectVersions +// operation +type ListObjectVersionsAPIClient interface { + ListObjectVersions(context.Context, *ListObjectVersionsInput, ...func(*Options)) (*ListObjectVersionsOutput, error) +} + +var _ ListObjectVersionsAPIClient = (*Client)(nil) + +// ListObjectVersionsPaginatorOptions is the paginator options for ListObjectVersions +type ListObjectVersionsPaginatorOptions struct { + // (Optional) The maximum number of Object Versions that you want Amazon S3 to + // return. + Limit int32 + + // Set to true if pagination should stop if the service returns a pagination token + // that matches the most recent token provided to the service. + StopOnDuplicateToken bool +} + +// ListObjectVersionsPaginator is a paginator for ListObjectVersions +type ListObjectVersionsPaginator struct { + options ListObjectVersionsPaginatorOptions + client ListObjectVersionsAPIClient + params *ListObjectVersionsInput + firstPage bool + keyMarker *string + versionIDMarker *string + isTruncated bool +} + +// NewListObjectVersionsPaginator returns a new ListObjectVersionsPaginator +func NewListObjectVersionsPaginator(client ListObjectVersionsAPIClient, params *ListObjectVersionsInput, optFns ...func(*ListObjectVersionsPaginatorOptions)) *ListObjectVersionsPaginator { + if params == nil { + params = &ListObjectVersionsInput{} + } + + options := ListObjectVersionsPaginatorOptions{} + options.Limit = params.MaxKeys + + for _, fn := range optFns { + fn(&options) + } + + return &ListObjectVersionsPaginator{ + options: options, + client: client, + params: params, + firstPage: true, + keyMarker: params.KeyMarker, + versionIDMarker: params.VersionIdMarker, + } +} + +// HasMorePages returns a boolean indicating whether more pages are available +func (p *ListObjectVersionsPaginator) HasMorePages() bool { + return p.firstPage || p.isTruncated +} + +// NextPage retrieves the next ListObjectVersions page. +func (p *ListObjectVersionsPaginator) NextPage(ctx context.Context, optFns ...func(*Options)) (*ListObjectVersionsOutput, error) { + if !p.HasMorePages() { + return nil, fmt.Errorf("no more pages available") + } + + params := *p.params + params.KeyMarker = p.keyMarker + params.VersionIdMarker = p.versionIDMarker + + var limit int32 + if p.options.Limit > 0 { + limit = p.options.Limit + } + params.MaxKeys = limit + + result, err := p.client.ListObjectVersions(ctx, ¶ms, optFns...) + if err != nil { + return nil, err + } + p.firstPage = false + + prevToken := p.keyMarker + p.isTruncated = result.IsTruncated + p.keyMarker = nil + p.versionIDMarker = nil + if result.IsTruncated { + p.keyMarker = result.NextKeyMarker + p.versionIDMarker = result.NextVersionIdMarker + } + + if p.options.StopOnDuplicateToken && + prevToken != nil && + p.keyMarker != nil && + *prevToken == *p.keyMarker { + p.isTruncated = false + } + + return result, nil +} + +// ListMultipartUploadsAPIClient is a client that implements the ListMultipartUploads +// operation +type ListMultipartUploadsAPIClient interface { + ListMultipartUploads(context.Context, *ListMultipartUploadsInput, ...func(*Options)) (*ListMultipartUploadsOutput, error) +} + +var _ ListMultipartUploadsAPIClient = (*Client)(nil) + +// ListMultipartUploadsPaginatorOptions is the paginator options for ListMultipartUploads +type ListMultipartUploadsPaginatorOptions struct { + // (Optional) The maximum number of Multipart Uploads that you want Amazon S3 to + // return. + Limit int32 + + // Set to true if pagination should stop if the service returns a pagination token + // that matches the most recent token provided to the service. + StopOnDuplicateToken bool +} + +// ListMultipartUploadsPaginator is a paginator for ListMultipartUploads +type ListMultipartUploadsPaginator struct { + options ListMultipartUploadsPaginatorOptions + client ListMultipartUploadsAPIClient + params *ListMultipartUploadsInput + firstPage bool + keyMarker *string + uploadIDMarker *string + isTruncated bool +} + +// NewListMultipartUploadsPaginator returns a new ListMultipartUploadsPaginator +func NewListMultipartUploadsPaginator(client ListMultipartUploadsAPIClient, params *ListMultipartUploadsInput, optFns ...func(*ListMultipartUploadsPaginatorOptions)) *ListMultipartUploadsPaginator { + if params == nil { + params = &ListMultipartUploadsInput{} + } + + options := ListMultipartUploadsPaginatorOptions{} + options.Limit = params.MaxUploads + + for _, fn := range optFns { + fn(&options) + } + + return &ListMultipartUploadsPaginator{ + options: options, + client: client, + params: params, + firstPage: true, + keyMarker: params.KeyMarker, + uploadIDMarker: params.UploadIdMarker, + } +} + +// HasMorePages returns a boolean indicating whether more pages are available +func (p *ListMultipartUploadsPaginator) HasMorePages() bool { + return p.firstPage || p.isTruncated +} + +// NextPage retrieves the next ListMultipartUploads page. +func (p *ListMultipartUploadsPaginator) NextPage(ctx context.Context, optFns ...func(*Options)) (*ListMultipartUploadsOutput, error) { + if !p.HasMorePages() { + return nil, fmt.Errorf("no more pages available") + } + + params := *p.params + params.KeyMarker = p.keyMarker + params.UploadIdMarker = p.uploadIDMarker + + var limit int32 + if p.options.Limit > 0 { + limit = p.options.Limit + } + params.MaxUploads = limit + + result, err := p.client.ListMultipartUploads(ctx, ¶ms, optFns...) + if err != nil { + return nil, err + } + p.firstPage = false + + prevToken := p.keyMarker + p.isTruncated = result.IsTruncated + p.keyMarker = nil + p.uploadIDMarker = nil + if result.IsTruncated { + p.keyMarker = result.NextKeyMarker + p.uploadIDMarker = result.NextUploadIdMarker + } + + if p.options.StopOnDuplicateToken && + prevToken != nil && + p.keyMarker != nil && + *prevToken == *p.keyMarker { + p.isTruncated = false + } + + return result, nil +} diff --git a/service/s3/handwritten_paginators_test.go b/service/s3/handwritten_paginators_test.go new file mode 100644 index 00000000000..6ded64edd2d --- /dev/null +++ b/service/s3/handwritten_paginators_test.go @@ -0,0 +1,281 @@ +package s3 + +import ( + "context" + "github.com/aws/aws-sdk-go-v2/aws" + "testing" +) + +type mockListObjectVersionsClient struct { + outputs []*ListObjectVersionsOutput + inputs []*ListObjectVersionsInput + t *testing.T +} + +type mockListMultipartUploadsClient struct { + outputs []*ListMultipartUploadsOutput + inputs []*ListMultipartUploadsInput + t *testing.T +} + +func (c *mockListObjectVersionsClient) ListObjectVersions(ctx context.Context, input *ListObjectVersionsInput, optFns ...func(*Options)) (*ListObjectVersionsOutput, error) { + c.inputs = append(c.inputs, input) + requestCnt := len(c.inputs) + testCurRequest(len(c.outputs), requestCnt, c.outputs[requestCnt-1].MaxKeys, input.MaxKeys, c.t) + return c.outputs[requestCnt-1], nil +} + +func (c *mockListMultipartUploadsClient) ListMultipartUploads(ctx context.Context, input *ListMultipartUploadsInput, optFns ...func(*Options)) (*ListMultipartUploadsOutput, error) { + c.inputs = append(c.inputs, input) + requestCnt := len(c.inputs) + testCurRequest(len(c.outputs), requestCnt, c.outputs[requestCnt-1].MaxUploads, input.MaxUploads, c.t) + return c.outputs[requestCnt-1], nil +} + +type testCase struct { + bucket *string + limit int32 + requestCnt int + stopOnDuplicationToken bool +} + +type listOVTestCase struct { + testCase + outputs []*ListObjectVersionsOutput +} + +type listMPUTestCase struct { + testCase + outputs []*ListMultipartUploadsOutput +} + +func TestListObjectVersionsPaginator(t *testing.T) { + cases := map[string]listOVTestCase{ + "page limit 5": { + testCase: testCase{ + bucket: aws.String("testBucket1"), + limit: 5, + requestCnt: 3, + }, + outputs: []*ListObjectVersionsOutput{ + &ListObjectVersionsOutput{ + NextKeyMarker: aws.String("testKey1"), + NextVersionIdMarker: aws.String("testID1"), + MaxKeys: 5, + IsTruncated: true, + }, + &ListObjectVersionsOutput{ + NextKeyMarker: aws.String("testKey2"), + NextVersionIdMarker: aws.String("testID2"), + MaxKeys: 5, + IsTruncated: true, + }, + &ListObjectVersionsOutput{ + NextKeyMarker: aws.String("testKey3"), + NextVersionIdMarker: aws.String("testID3"), + MaxKeys: 5, + IsTruncated: false, + }, + }, + }, + "page limit 10 with duplicate marker": { + testCase: testCase{ + bucket: aws.String("testBucket2"), + limit: 10, + requestCnt: 3, + stopOnDuplicationToken: true, + }, + outputs: []*ListObjectVersionsOutput{ + &ListObjectVersionsOutput{ + NextKeyMarker: aws.String("testKey1"), + NextVersionIdMarker: aws.String("testID1"), + MaxKeys: 10, + IsTruncated: true, + }, + &ListObjectVersionsOutput{ + NextKeyMarker: aws.String("testKey2"), + NextVersionIdMarker: aws.String("testID2"), + MaxKeys: 10, + IsTruncated: true, + }, + &ListObjectVersionsOutput{ + NextKeyMarker: aws.String("testKey2"), + NextVersionIdMarker: aws.String("testID2"), + MaxKeys: 10, + IsTruncated: true, + }, + &ListObjectVersionsOutput{ + NextKeyMarker: aws.String("testKey3"), + NextVersionIdMarker: aws.String("testID3"), + MaxKeys: 10, + IsTruncated: false, + }, + }, + }, + } + + for name, c := range cases { + t.Run(name, func(t *testing.T) { + client := mockListObjectVersionsClient{ + t: t, + outputs: c.outputs, + inputs: []*ListObjectVersionsInput{}, + } + paginator := NewListObjectVersionsPaginator(&client, &ListObjectVersionsInput{ + Bucket: c.bucket, + }, func(options *ListObjectVersionsPaginatorOptions) { + options.Limit = c.limit + options.StopOnDuplicateToken = c.stopOnDuplicationToken + }) + + for paginator.HasMorePages() { + _, err := paginator.NextPage(context.TODO()) + if err != nil { + t.Errorf("error: %v", err) + } + } + + inputLen := len(client.inputs) + testTotalRequests(c.requestCnt, inputLen, t) + for i := 1; i < inputLen; i++ { + if *client.inputs[i].KeyMarker != *c.outputs[i-1].NextKeyMarker { + t.Errorf("Expect next input's KeyMarker to be eaqul to %s, got %s", + *c.outputs[i-1].NextKeyMarker, *client.inputs[i].KeyMarker) + } + if *client.inputs[i].VersionIdMarker != *c.outputs[i-1].NextVersionIdMarker { + t.Errorf("Expect next input's VersionIdMarker to be eaqul to %s, got %s", + *c.outputs[i-1].NextVersionIdMarker, *client.inputs[i].VersionIdMarker) + } + } + }) + } +} + +func TestListMultipartUploadsPaginator(t *testing.T) { + cases := map[string]listMPUTestCase{ + "page limit 5": { + testCase: testCase{ + bucket: aws.String("testBucket1"), + limit: 5, + requestCnt: 4, + }, + outputs: []*ListMultipartUploadsOutput{ + &ListMultipartUploadsOutput{ + NextKeyMarker: aws.String("testKey1"), + NextUploadIdMarker: aws.String("testID1"), + MaxUploads: 5, + IsTruncated: true, + }, + &ListMultipartUploadsOutput{ + NextKeyMarker: aws.String("testKey2"), + NextUploadIdMarker: aws.String("testID2"), + MaxUploads: 5, + IsTruncated: true, + }, + &ListMultipartUploadsOutput{ + NextKeyMarker: aws.String("testKey3"), + NextUploadIdMarker: aws.String("testID3"), + MaxUploads: 5, + IsTruncated: true, + }, + &ListMultipartUploadsOutput{ + NextKeyMarker: aws.String("testKey4"), + NextUploadIdMarker: aws.String("testID4"), + MaxUploads: 5, + IsTruncated: false, + }, + }, + }, + "page limit 10 with duplicate marker": { + testCase: testCase{ + bucket: aws.String("testBucket2"), + limit: 10, + requestCnt: 3, + stopOnDuplicationToken: true, + }, + outputs: []*ListMultipartUploadsOutput{ + &ListMultipartUploadsOutput{ + NextKeyMarker: aws.String("testKey1"), + NextUploadIdMarker: aws.String("testID1"), + MaxUploads: 10, + IsTruncated: true, + }, + &ListMultipartUploadsOutput{ + NextKeyMarker: aws.String("testKey2"), + NextUploadIdMarker: aws.String("testID2"), + MaxUploads: 10, + IsTruncated: true, + }, + &ListMultipartUploadsOutput{ + NextKeyMarker: aws.String("testKey2"), + NextUploadIdMarker: aws.String("testID2"), + MaxUploads: 10, + IsTruncated: true, + }, + &ListMultipartUploadsOutput{ + NextKeyMarker: aws.String("testKey4"), + NextUploadIdMarker: aws.String("testID4"), + MaxUploads: 10, + IsTruncated: false, + }, + &ListMultipartUploadsOutput{ + NextKeyMarker: aws.String("testKey5"), + NextUploadIdMarker: aws.String("testID5"), + MaxUploads: 10, + IsTruncated: false, + }, + }, + }, + } + + for name, c := range cases { + t.Run(name, func(t *testing.T) { + client := mockListMultipartUploadsClient{ + outputs: c.outputs, + inputs: []*ListMultipartUploadsInput{}, + t: t, + } + paginator := NewListMultipartUploadsPaginator(&client, &ListMultipartUploadsInput{ + Bucket: c.bucket, + }, func(options *ListMultipartUploadsPaginatorOptions) { + options.Limit = c.limit + options.StopOnDuplicateToken = c.stopOnDuplicationToken + }) + + for paginator.HasMorePages() { + _, err := paginator.NextPage(context.TODO()) + if err != nil { + t.Errorf("error: %v", err) + } + } + + inputLen := len(client.inputs) + testTotalRequests(c.requestCnt, inputLen, t) + for i := 1; i < inputLen; i++ { + if *client.inputs[i].KeyMarker != *c.outputs[i-1].NextKeyMarker { + t.Errorf("Expect next input's KeyMarker to be eaqul to %s, got %s", + *c.outputs[i-1].NextKeyMarker, *client.inputs[i].KeyMarker) + } + if *client.inputs[i].UploadIdMarker != *c.outputs[i-1].NextUploadIdMarker { + t.Errorf("Expect next input's UploadIdMarker to be eaqul to %s, got %s", + *c.outputs[i-1].NextUploadIdMarker, *client.inputs[i].UploadIdMarker) + } + } + }) + } +} + +func testTotalRequests(expect, actual int, t *testing.T) { + if actual != expect { + t.Errorf("Expect total request number to be %d, got %d", expect, actual) + } +} + +func testCurRequest(maxReqCnt, actualReqCnt int, expectLimit, actualLimit int32, t *testing.T) { + if actualReqCnt > maxReqCnt { + t.Errorf("Paginator calls client more than expected %d times", maxReqCnt) + } + if expectLimit != actualLimit { + t.Errorf("Expect page limit to be %d, got %d", expectLimit, actualLimit) + } +}