comments

go-to-k · Apr 13, 2024 · 8b70dbc · 8b70dbc
1 parent 8406e8c
commit 8b70dbc
Show file tree

Hide file tree

Showing 2 changed files with 41 additions and 5 deletions.
diff --git a/internal/wrapper/s3_wrapper.go b/internal/wrapper/s3_wrapper.go
@@ -62,6 +62,8 @@ func (s *S3Wrapper) ClearS3Objects(
 	for {
 		var versions []types.ObjectIdentifier
 
+		// ListObjectVersions API can only retrieve up to 1000 items, so it is good to pass it
+		// directly to DeleteObjects, which can only delete up to 1000 items.
 		versions, keyMarker, versionIdMarker, err = s.client.ListObjectVersionsByPage(
 			ctx,
 			aws.String(bucketName),
@@ -82,6 +84,10 @@ func (s *S3Wrapper) ClearS3Objects(
 		}
 
 		eg.Go(func() error {
+			// One DeleteObjects is executed for each loop of the List, and it usually ends during
+			// the next loop. Therefore, there seems to be no throttling concern, so the number of
+			// parallels is not limited by semaphore. (Throttling occurs at about 3500 deletions
+			// per second.)
 			gotErrors, err := s.client.DeleteObjects(ctx, aws.String(bucketName), versions, region)
 			if err != nil {
 				return err

diff --git a/pkg/client/s3.go b/pkg/client/s3.go
@@ -14,8 +14,17 @@ var SleepTimeSecForS3 = 10
 
 type IS3 interface {
 	DeleteBucket(ctx context.Context, bucketName *string, region string) error
-	DeleteObjects(ctx context.Context, bucketName *string, objects []types.ObjectIdentifier, region string) ([]types.Error, error)
-	ListObjectVersions(ctx context.Context, bucketName *string, region string, oldVersionsOnly bool) ([]types.ObjectIdentifier, error)
+	DeleteObjects(
+		ctx context.Context,
+		bucketName *string,
+		objects []types.ObjectIdentifier,
+		region string,
+	) ([]types.Error, error)
+	ListObjectVersions(ctx context.Context,
+		bucketName *string,
+		region string,
+		oldVersionsOnly bool,
+	) ([]types.ObjectIdentifier, error)
 	ListObjectVersionsByPage(
 		ctx context.Context,
 		bucketName *string,
@@ -63,7 +72,15 @@ func (s *S3) DeleteBucket(ctx context.Context, bucketName *string, region string
 	return nil
 }
 
-func (s *S3) DeleteObjects(ctx context.Context, bucketName *string, objects []types.ObjectIdentifier, region string) ([]types.Error, error) {
+func (s *S3) DeleteObjects(
+	ctx context.Context,
+	bucketName *string,
+	objects []types.ObjectIdentifier,
+	region string,
+) ([]types.Error, error) {
+	// Assuming that the number of objects received as an argument does not
+	// exceed 1000, so no loop processing and validation whether exceeds
+	// 1000 or not are good.
 	if len(objects) == 0 {
 		return []types.Error{}, nil
 	}
@@ -95,7 +112,12 @@ func (s *S3) DeleteObjects(ctx context.Context, bucketName *string, objects []ty
 	return output.Errors, nil
 }
 
-func (s *S3) ListObjectVersions(ctx context.Context, bucketName *string, region string, oldVersionsOnly bool) ([]types.ObjectIdentifier, error) {
+func (s *S3) ListObjectVersions(
+	ctx context.Context,
+	bucketName *string,
+	region string,
+	oldVersionsOnly bool,
+) ([]types.ObjectIdentifier, error) {
 	var keyMarker *string
 	var versionIdMarker *string
 	objectIdentifiers := []types.ObjectIdentifier{}
@@ -110,7 +132,15 @@ func (s *S3) ListObjectVersions(ctx context.Context, bucketName *string, region
 		default:
 		}
 
-		objectIdentifiersByPage, nextKeyMarker, nextVersionIdMarker, err := s.ListObjectVersionsByPage(ctx, bucketName, region, oldVersionsOnly, keyMarker, versionIdMarker)
+		objectIdentifiersByPage, nextKeyMarker, nextVersionIdMarker, err :=
+			s.ListObjectVersionsByPage(
+				ctx,
+				bucketName,
+				region,
+				oldVersionsOnly,
+				keyMarker,
+				versionIdMarker,
+			)
 		if err != nil {
 			return nil, err // ListObjectVersionsByPage already wraps the error
 		}