fix: s3 upload exceeded total allowed configured MaxUploadParts (close …

…#5909)
alist-org · Jan 19, 2024 · 0f29a81 · 0f29a81
1 parent 442c2f7
commit 0f29a81
Show file tree

Hide file tree

Showing 7 changed files with 25 additions and 5 deletions.
diff --git a/drivers/123/driver.go b/drivers/123/driver.go
@@ -6,6 +6,10 @@ import (
 	"encoding/base64"
 	"encoding/hex"
 	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+
 	"github.com/alist-org/alist/v3/drivers/base"
 	"github.com/alist-org/alist/v3/internal/driver"
 	"github.com/alist-org/alist/v3/internal/errs"
@@ -17,9 +21,6 @@ import (
 	"github.com/aws/aws-sdk-go/service/s3/s3manager"
 	"github.com/go-resty/resty/v2"
 	log "github.com/sirupsen/logrus"
-	"io"
-	"net/http"
-	"net/url"
 )
 
 type Pan123 struct {
@@ -232,6 +233,9 @@ func (d *Pan123) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr
 			return err
 		}
 		uploader := s3manager.NewUploader(s)
+		if stream.GetSize() > s3manager.MaxUploadParts*s3manager.DefaultUploadPartSize {
+			uploader.PartSize = stream.GetSize() / (s3manager.MaxUploadParts - 1)
+		}
 		input := &s3manager.UploadInput{
 			Bucket: &resp.Data.Bucket,
 			Key:    &resp.Data.Key,

diff --git a/drivers/mediatrack/driver.go b/drivers/mediatrack/driver.go
@@ -188,6 +188,9 @@ func (d *MediaTrack) Put(ctx context.Context, dstDir model.Obj, stream model.Fil
 		_ = tempFile.Close()
 	}()
 	uploader := s3manager.NewUploader(s)
+	if stream.GetSize() > s3manager.MaxUploadParts*s3manager.DefaultUploadPartSize {
+		uploader.PartSize = stream.GetSize() / (s3manager.MaxUploadParts - 1)
+	}
 	input := &s3manager.UploadInput{
 		Bucket: &resp.Data.Bucket,
 		Key:    &resp.Data.Object,

diff --git a/drivers/pikpak/driver.go b/drivers/pikpak/driver.go
@@ -172,6 +172,9 @@ func (d *PikPak) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr
 		return err
 	}
 	uploader := s3manager.NewUploader(ss)
+	if stream.GetSize() > s3manager.MaxUploadParts*s3manager.DefaultUploadPartSize {
+		uploader.PartSize = stream.GetSize() / (s3manager.MaxUploadParts - 1)
+	}
 	input := &s3manager.UploadInput{
 		Bucket: &params.Bucket,
 		Key:    &params.Key,

diff --git a/drivers/quqi/driver.go b/drivers/quqi/driver.go
@@ -311,7 +311,7 @@ func (d *Quqi) Put(ctx context.Context, dstDir model.Obj, stream model.FileStrea
 			length = stream.GetSize() - (int64(i)-1)*partSize
 		}
 		_, err := client.Object.UploadPart(
-			context.Background(), uploadInitResp.Data.Key, uploadInitResp.Data.UploadID, i, io.LimitReader(f, partSize), &cos.ObjectUploadPartOptions{
+			ctx, uploadInitResp.Data.Key, uploadInitResp.Data.UploadID, i, io.LimitReader(f, partSize), &cos.ObjectUploadPartOptions{
 				ContentLength: length,
 			},
 		)

diff --git a/drivers/teambition/util.go b/drivers/teambition/util.go
@@ -244,6 +244,9 @@ func (d *Teambition) newUpload(ctx context.Context, dstDir model.Obj, stream mod
 		return err
 	}
 	uploader := s3manager.NewUploader(ss)
+	if stream.GetSize() > s3manager.MaxUploadParts*s3manager.DefaultUploadPartSize {
+		uploader.PartSize = stream.GetSize() / (s3manager.MaxUploadParts - 1)
+	}
 	input := &s3manager.UploadInput{
 		Bucket:             &uploadToken.Upload.Bucket,
 		Key:                &uploadToken.Upload.Key,

diff --git a/drivers/thunder/driver.go b/drivers/thunder/driver.go
@@ -373,7 +373,11 @@ func (xc *XunLeiCommon) Put(ctx context.Context, dstDir model.Obj, stream model.
 		if err != nil {
 			return err
 		}
-		_, err = s3manager.NewUploader(s).UploadWithContext(ctx, &s3manager.UploadInput{
+		uploader := s3manager.NewUploader(s)
+		if stream.GetSize() > s3manager.MaxUploadParts*s3manager.DefaultUploadPartSize {
+			uploader.PartSize = stream.GetSize() / (s3manager.MaxUploadParts - 1)
+		}
+		_, err = uploader.UploadWithContext(ctx, &s3manager.UploadInput{
 			Bucket:  aws.String(param.Bucket),
 			Key:     aws.String(param.Key),
 			Expires: aws.Time(param.Expiration),

diff --git a/drivers/vtencent/util.go b/drivers/vtencent/util.go
@@ -272,6 +272,9 @@ func (d *Vtencent) FileUpload(ctx context.Context, dstDir model.Obj, stream mode
 		return err
 	}
 	uploader := s3manager.NewUploader(ss)
+	if stream.GetSize() > s3manager.MaxUploadParts*s3manager.DefaultUploadPartSize {
+		uploader.PartSize = stream.GetSize() / (s3manager.MaxUploadParts - 1)
+	}
 	input := &s3manager.UploadInput{
 		Bucket: aws.String(fmt.Sprintf("%s-%d", params.StorageBucket, params.StorageAppID)),
 		Key:    &params.Video.StoragePath,