feat(s3): server support generated url request (#6431)

go.mod

@@ -3,11 +3,11 @@ module github.com/alist-org/alist/v3
 go 1.21
 
 require (
-	github.com/Mikubill/gofakes3 v0.0.3-0.20230622102024-284c0f988700
 	github.com/SheltonZhu/115driver v1.0.22
 	github.com/Xhofe/go-cache v0.0.0-20220723083548-714439c8af9a
 	github.com/Xhofe/rateg v0.0.0-20230728072201-251a4e1adad4
 	github.com/Xhofe/wopan-sdk-go v0.1.2
+	github.com/alist-org/gofakes3 v0.0.4
 	github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible
 	github.com/avast/retry-go v3.0.0+incompatible
 	github.com/aws/aws-sdk-go v1.50.24

go.sum

@@ -7,8 +7,6 @@ github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd h1:nzE1YQBdx1bq9IlZinHa+HVffy+NmVRoKr+wHN8fpLE=
 github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd/go.mod h1:C8yoIfvESpM3GD07OCHU7fqI7lhwyZ2Td1rbNbTAhnc=
-github.com/Mikubill/gofakes3 v0.0.3-0.20230622102024-284c0f988700 h1:r3fp2/Ro+0RtpjNY0/wsbN7vRmCW//dXTOZDQTct25Q=
-github.com/Mikubill/gofakes3 v0.0.3-0.20230622102024-284c0f988700/go.mod h1:OSXqXEGUe9CmPiwLMMnVrbXonMf4BeLBkBdLufxxiyY=
 github.com/RoaringBitmap/roaring v1.2.3 h1:yqreLINqIrX22ErkKI0vY47/ivtJr6n+kMhVOVmhWBY=
 github.com/RoaringBitmap/roaring v1.2.3/go.mod h1:plvDsJQpxOC5bw8LRteu/MLWHsHez/3y6cubLI4/1yE=
 github.com/SheltonZhu/115driver v1.0.22 h1:Wp8pN7/gK3YwEO5P18ggbIOHM++lo9eP/pBhuvXfI6U=
@@ -25,6 +23,8 @@ github.com/abbot/go-http-auth v0.4.0 h1:QjmvZ5gSC7jm3Zg54DqWE/T5m1t2AfDu6QlXJT0E
 github.com/abbot/go-http-auth v0.4.0/go.mod h1:Cz6ARTIzApMJDzh5bRMSUou6UMSp0IEXg9km/ci7TJM=
 github.com/aead/ecdh v0.2.0 h1:pYop54xVaq/CEREFEcukHRZfTdjiWvYIsZDXXrBapQQ=
 github.com/aead/ecdh v0.2.0/go.mod h1:a9HHtXuSo8J1Js1MwLQx2mBhkXMT6YwUmVVEY4tTB8U=
+github.com/alist-org/gofakes3 v0.0.4 h1:/ID4+1llsiB8EweLcC65rVmgBZKL95e3P7Wa+aJGUiE=
+github.com/alist-org/gofakes3 v0.0.4/go.mod h1:bLPZXt45XYMgaoGGLe5t0d1p13oZTQTptTEDLrku070=
 github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible h1:8psS8a+wKfiLt1iVDX79F7Y6wUM49Lcha2FMXt4UM8g=
 github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8=
 github.com/andreburgaud/crypt2go v1.2.0 h1:oly/ENAodeqTYpUafgd4r3v+VKLQnmOKUyfpj+TxHbE=

server/s3/backend.go

@@ -12,20 +12,20 @@ import (
 	"sync"
 	"time"
 
-	"github.com/Mikubill/gofakes3"
 	"github.com/alist-org/alist/v3/internal/errs"
 	"github.com/alist-org/alist/v3/internal/fs"
 	"github.com/alist-org/alist/v3/internal/model"
 	"github.com/alist-org/alist/v3/internal/op"
 	"github.com/alist-org/alist/v3/internal/stream"
 	"github.com/alist-org/alist/v3/pkg/http_range"
 	"github.com/alist-org/alist/v3/pkg/utils"
+	"github.com/alist-org/gofakes3"
 	"github.com/ncw/swift/v2"
 )
 
 var (
 	emptyPrefix = &gofakes3.Prefix{}
-	timeFormat  = "Mon, 2 Jan 2006 15:04:05.999999999 GMT"
+	timeFormat  = "Mon, 2 Jan 2006 15:04:05 GMT"
 )
 
 // s3Backend implements the gofacess3.Backend interface to make an S3
@@ -42,13 +42,12 @@ func newBackend() gofakes3.Backend {
 }
 
 // ListBuckets always returns the default bucket.
-func (b *s3Backend) ListBuckets() ([]gofakes3.BucketInfo, error) {
+func (b *s3Backend) ListBuckets(ctx context.Context) ([]gofakes3.BucketInfo, error) {
 	buckets, err := getAndParseBuckets()
 	if err != nil {
 		return nil, err
 	}
 	var response []gofakes3.BucketInfo
-	ctx := context.Background()
 	for _, b := range buckets {
 		node, _ := fs.Get(ctx, b.Path, &fs.GetArgs{})
 		response = append(response, gofakes3.BucketInfo{
@@ -61,7 +60,7 @@ func (b *s3Backend) ListBuckets() ([]gofakes3.BucketInfo, error) {
 }
 
 // ListBucket lists the objects in the given bucket.
-func (b *s3Backend) ListBucket(bucketName string, prefix *gofakes3.Prefix, page gofakes3.ListBucketPage) (*gofakes3.ObjectList, error) {
+func (b *s3Backend) ListBucket(ctx context.Context, bucketName string, prefix *gofakes3.Prefix, page gofakes3.ListBucketPage) (*gofakes3.ObjectList, error) {
 	bucket, err := getBucketByName(bucketName)
 	if err != nil {
 		return nil, err
@@ -97,8 +96,7 @@ func (b *s3Backend) ListBucket(bucketName string, prefix *gofakes3.Prefix, page
 // HeadObject returns the fileinfo for the given object name.
 //
 // Note that the metadata is not supported yet.
-func (b *s3Backend) HeadObject(bucketName, objectName string) (*gofakes3.Object, error) {
-	ctx := context.Background()
+func (b *s3Backend) HeadObject(ctx context.Context, bucketName, objectName string) (*gofakes3.Object, error) {
 	bucket, err := getBucketByName(bucketName)
 	if err != nil {
 		return nil, err
@@ -141,8 +139,7 @@ func (b *s3Backend) HeadObject(bucketName, objectName string) (*gofakes3.Object,
 }
 
 // GetObject fetchs the object from the filesystem.
-func (b *s3Backend) GetObject(bucketName, objectName string, rangeRequest *gofakes3.ObjectRangeRequest) (obj *gofakes3.Object, err error) {
-	ctx := context.Background()
+func (b *s3Backend) GetObject(ctx context.Context, bucketName, objectName string, rangeRequest *gofakes3.ObjectRangeRequest) (obj *gofakes3.Object, err error) {
 	bucket, err := getBucketByName(bucketName)
 	if err != nil {
 		return nil, err
@@ -251,18 +248,17 @@ func (b *s3Backend) GetObject(bucketName, objectName string, rangeRequest *gofak
 }
 
 // TouchObject creates or updates meta on specified object.
-func (b *s3Backend) TouchObject(fp string, meta map[string]string) (result gofakes3.PutObjectResult, err error) {
+func (b *s3Backend) TouchObject(ctx context.Context, fp string, meta map[string]string) (result gofakes3.PutObjectResult, err error) {
 	//TODO: implement
 	return result, gofakes3.ErrNotImplemented
 }
 
 // PutObject creates or overwrites the object with the given name.
 func (b *s3Backend) PutObject(
-	bucketName, objectName string,
+	ctx context.Context, bucketName, objectName string,
 	meta map[string]string,
 	input io.Reader, size int64,
 ) (result gofakes3.PutObjectResult, err error) {
-	ctx := context.Background()
 	bucket, err := getBucketByName(bucketName)
 	if err != nil {
 		return result, err
@@ -316,9 +312,9 @@ func (b *s3Backend) PutObject(
 }
 
 // DeleteMulti deletes multiple objects in a single request.
-func (b *s3Backend) DeleteMulti(bucketName string, objects ...string) (result gofakes3.MultiDeleteResult, rerr error) {
+func (b *s3Backend) DeleteMulti(ctx context.Context, bucketName string, objects ...string) (result gofakes3.MultiDeleteResult, rerr error) {
 	for _, object := range objects {
-		if err := b.deleteObject(bucketName, object); err != nil {
+		if err := b.deleteObject(ctx, bucketName, object); err != nil {
 			utils.Log.Errorf("serve s3", "delete object failed: %v", err)
 			result.Error = append(result.Error, gofakes3.ErrorResult{
 				Code:    gofakes3.ErrInternal,
@@ -336,13 +332,12 @@ func (b *s3Backend) DeleteMulti(bucketName string, objects ...string) (result go
 }
 
 // DeleteObject deletes the object with the given name.
-func (b *s3Backend) DeleteObject(bucketName, objectName string) (result gofakes3.ObjectDeleteResult, rerr error) {
-	return result, b.deleteObject(bucketName, objectName)
+func (b *s3Backend) DeleteObject(ctx context.Context, bucketName, objectName string) (result gofakes3.ObjectDeleteResult, rerr error) {
+	return result, b.deleteObject(ctx, bucketName, objectName)
 }
 
 // deleteObject deletes the object from the filesystem.
-func (b *s3Backend) deleteObject(bucketName, objectName string) error {
-	ctx := context.Background()
+func (b *s3Backend) deleteObject(ctx context.Context, bucketName, objectName string) error {
 	bucket, err := getBucketByName(bucketName)
 	if err != nil {
 		return err
@@ -362,17 +357,17 @@ func (b *s3Backend) deleteObject(bucketName, objectName string) error {
 }
 
 // CreateBucket creates a new bucket.
-func (b *s3Backend) CreateBucket(name string) error {
+func (b *s3Backend) CreateBucket(ctx context.Context, name string) error {
 	return gofakes3.ErrNotImplemented
 }
 
 // DeleteBucket deletes the bucket with the given name.
-func (b *s3Backend) DeleteBucket(name string) error {
+func (b *s3Backend) DeleteBucket(ctx context.Context, name string) error {
 	return gofakes3.ErrNotImplemented
 }
 
 // BucketExists checks if the bucket exists.
-func (b *s3Backend) BucketExists(name string) (exists bool, err error) {
+func (b *s3Backend) BucketExists(ctx context.Context, name string) (exists bool, err error) {
 	buckets, err := getAndParseBuckets()
 	if err != nil {
 		return false, err
@@ -386,13 +381,12 @@ func (b *s3Backend) BucketExists(name string) (exists bool, err error) {
 }
 
 // CopyObject copy specified object from srcKey to dstKey.
-func (b *s3Backend) CopyObject(srcBucket, srcKey, dstBucket, dstKey string, meta map[string]string) (result gofakes3.CopyObjectResult, err error) {
+func (b *s3Backend) CopyObject(ctx context.Context, srcBucket, srcKey, dstBucket, dstKey string, meta map[string]string) (result gofakes3.CopyObjectResult, err error) {
 	if srcBucket == dstBucket && srcKey == dstKey {
 		//TODO: update meta
 		return result, nil
 	}
 
-	ctx := context.Background()
 	srcB, err := getBucketByName(srcBucket)
 	if err != nil {
 		return result, err
@@ -403,7 +397,7 @@ func (b *s3Backend) CopyObject(srcBucket, srcKey, dstBucket, dstKey string, meta
 	fmeta, _ := op.GetNearestMeta(srcFp)
 	srcNode, err := fs.Get(context.WithValue(ctx, "meta", fmeta), srcFp, &fs.GetArgs{})
 
-	c, err := b.GetObject(srcBucket, srcKey, nil)
+	c, err := b.GetObject(ctx, srcBucket, srcKey, nil)
 	if err != nil {
 		return
 	}
@@ -420,7 +414,7 @@ func (b *s3Backend) CopyObject(srcBucket, srcKey, dstBucket, dstKey string, meta
 		meta["mtime"] = swift.TimeToFloatString(srcNode.ModTime())
 	}
 
-	_, err = b.PutObject(dstBucket, dstKey, meta, c.Contents, c.Size)
+	_, err = b.PutObject(ctx, dstBucket, dstKey, meta, c.Contents, c.Size)
 	if err != nil {
 		return
 	}

server/s3/list.go

@@ -6,7 +6,7 @@ import (
 	"path"
 	"strings"
 
-	"github.com/Mikubill/gofakes3"
+	"github.com/alist-org/gofakes3"
 )
 
 func (b *s3Backend) entryListR(bucket, fdPath, name string, addPrefix bool, response *gofakes3.ObjectList) error {

server/s3/logger.go

@@ -5,8 +5,8 @@ package s3
 import (
 	"fmt"
 
-	"github.com/Mikubill/gofakes3"
 	"github.com/alist-org/alist/v3/pkg/utils"
+	"github.com/alist-org/gofakes3"
 )
 
 // logger output formatted message

server/s3/pager.go

@@ -5,7 +5,7 @@ package s3
 import (
 	"sort"
 
-	"github.com/Mikubill/gofakes3"
+	"github.com/alist-org/gofakes3"
 )
 
 // pager splits the object list into smulitply pages.

server/s3/server.go

@@ -7,7 +7,7 @@ import (
 	"math/rand"
 	"net/http"
 
-	"github.com/Mikubill/gofakes3"
+	"github.com/alist-org/gofakes3"
 )
 
 // Make a new S3 Server to serve the remote

server/s3/utils.go

@@ -7,13 +7,13 @@ import (
 	"encoding/json"
 	"strings"
 
-	"github.com/Mikubill/gofakes3"
 	"github.com/alist-org/alist/v3/internal/conf"
 	"github.com/alist-org/alist/v3/internal/errs"
 	"github.com/alist-org/alist/v3/internal/fs"
 	"github.com/alist-org/alist/v3/internal/model"
 	"github.com/alist-org/alist/v3/internal/op"
 	"github.com/alist-org/alist/v3/internal/setting"
+	"github.com/alist-org/gofakes3"
 )
 
 type Bucket struct {