From 473b53dee177435376a0385810c047ccb2abf39f Mon Sep 17 00:00:00 2001 From: saltbo Date: Sun, 17 May 2020 22:56:50 +0800 Subject: [PATCH] refactor: use the s3 replace the cloud platform sdk --- .gitignore | 3 +- core/engine.go | 23 ++--------- scripts/driverenv.sh | 21 ++++++++++ uploader/cos.go | 71 -------------------------------- uploader/driver.go | 32 +++++++++------ uploader/driver_test.go | 26 ++++++++---- uploader/google.go | 91 ----------------------------------------- uploader/oss.go | 62 ---------------------------- uploader/qiniu.go | 88 --------------------------------------- uploader/s3.go | 13 +++--- utils/utils.go | 23 +++++++++++ 11 files changed, 93 insertions(+), 360 deletions(-) create mode 100644 scripts/driverenv.sh delete mode 100644 uploader/cos.go delete mode 100644 uploader/google.go delete mode 100644 uploader/oss.go delete mode 100644 uploader/qiniu.go create mode 100644 utils/utils.go diff --git a/.gitignore b/.gitignore index a515fd3..3da21bd 100644 --- a/.gitignore +++ b/.gitignore @@ -14,4 +14,5 @@ # Dependency directories (remove the comment below to include it) vendor/ build/ -.coverprofile \ No newline at end of file +.coverprofile +.driverenvs.sh \ No newline at end of file diff --git a/core/engine.go b/core/engine.go index ab19605..c048a2a 100644 --- a/core/engine.go +++ b/core/engine.go @@ -1,15 +1,13 @@ package core import ( - "crypto/md5" - "encoding/hex" - "io" "log" "os" "path/filepath" "strings" "uptoc/uploader" + "uptoc/utils" ) // Engine is the core to finish the logic @@ -92,21 +90,6 @@ func (e *Engine) Sync() error { return nil } -func fileMD5(filepath string) string { - f, err := os.Open(filepath) - if err != nil { - return "" - } - defer f.Close() - - md5hash := md5.New() - if _, err := io.Copy(md5hash, f); err != nil { - return "" - } - - return hex.EncodeToString(md5hash.Sum(nil)[:]) -} - func objectExist(object uploader.Object, objects []uploader.Object) bool { for _, obj := range objects { if obj.Key == object.Key { @@ -118,7 +101,7 @@ func objectExist(object uploader.Object, objects []uploader.Object) bool { func objectNotMatch(object uploader.Object, objects []uploader.Object) bool { for _, obj := range objects { - if obj.Key == object.Key && obj.ETag != object.ETag { + if obj.Key == object.Key && strings.ToLower(obj.ETag) != object.ETag { return true } } @@ -134,7 +117,7 @@ func loadLocalObjects(dirPath string) ([]uploader.Object, error) { localObjects = append(localObjects, uploader.Object{ Key: strings.TrimPrefix(filePath, dirPath), - ETag: fileMD5(filePath), + ETag: utils.FileMD5(filePath), FilePath: filePath, }) return nil diff --git a/scripts/driverenv.sh b/scripts/driverenv.sh new file mode 100644 index 0000000..e87fb68 --- /dev/null +++ b/scripts/driverenv.sh @@ -0,0 +1,21 @@ +#!/bin/sh -l + +# aliyun oss +export UPLOADER_OSS_AK=LTAI4***********nL7YoV +export UPLOADER_OSS_SK=PFGVw**********************72Rd09u + +# tencent cos +export UPLOADER_COS_SK=GH0ScC***********OWY2Cn8H +export UPLOADER_COS_AK=AKIDTh**********************DWYKn1h5 + +# qiniu +export UPLOADER_QINIU_AK=dpFVS1rXn**********************9zSPwLs +export UPLOADER_QINIU_SK=AY4DV*********************************F9JfeFoyE + +# aws s3 +export UPLOADER_S3_AK=AKIA***********AA +export UPLOADER_S3_SK=/UGCFwk**********************bZmx9n + +# google storage +export UPLOADER_STORAGE_AK=GOOG***********VPM7Q +export UPLOADER_STORAGE_SK=bHw**********************wUIMnCb8 \ No newline at end of file diff --git a/uploader/cos.go b/uploader/cos.go deleted file mode 100644 index 7c6da86..0000000 --- a/uploader/cos.go +++ /dev/null @@ -1,71 +0,0 @@ -package uploader - -import ( - "context" - "fmt" - "net/http" - "net/url" - "strings" - - "github.com/tencentyun/cos-go-sdk-v5" -) - -// COSUploader implements the Driver base on tencent's cos. -type COSUploader struct { - *cos.Client -} - -// NewCOSUploader returns a new COS uploader -func NewCOSUploader(endpoint, accessKeyID, accessKeySecret, bucketName string) (Driver, error) { - u, err := url.Parse(fmt.Sprintf("https://%s.cos.%s.myqcloud.com", bucketName, endpoint)) - if err != nil { - return nil, err - } - - client := cos.NewClient(&cos.BaseURL{BucketURL: u}, &http.Client{ - Transport: &cos.AuthorizationTransport{ - SecretID: accessKeyID, - SecretKey: accessKeySecret, - }, - }) - - return &COSUploader{ - Client: client, - }, nil -} - -// ListObjects returns some remote objects -func (u *COSUploader) ListObjects() ([]Object, error) { - marker := "" - objects := make([]Object, 0) - for { - objectsResult, _, err := u.Bucket.Get(context.Background(), &cos.BucketGetOptions{Marker: marker}) - if err != nil { - return nil, err - } - - for _, obj := range objectsResult.Contents { - objects = append(objects, Object{Key: obj.Key, ETag: strings.ToLower(strings.Trim(obj.ETag, `"`))}) - } - - if objectsResult.IsTruncated { - marker = objectsResult.NextMarker - } else { - break - } - } - - return objects, nil -} - -// Upload uploads the local file to the object -func (u *COSUploader) Upload(object, rawPath string) (err error) { - _, err = u.Object.PutFromFile(context.Background(), object, rawPath, nil) - return -} - -// Delete deletes the object -func (u *COSUploader) Delete(object string) (err error) { - _, err = u.Object.Delete(context.Background(), object) - return -} diff --git a/uploader/driver.go b/uploader/driver.go index a68746f..f2d373c 100644 --- a/uploader/driver.go +++ b/uploader/driver.go @@ -2,6 +2,7 @@ package uploader import ( "fmt" + "strings" ) const ( @@ -28,22 +29,27 @@ type Driver interface { Delete(object string) error } -// Constructor defines the upload driver constructor used by some engine -type Constructor func(endpoint, accessKeyID, accessKeySecret, bucketName string) (Driver, error) - -var supportDrivers = map[string]Constructor{ - "oss": NewOSSUploader, - "cos": NewCOSUploader, - "qiniu": NewQiniuUploader, - "s3": NewS3Uploader, - "google": NewGoogleUploader, +var supportDrivers = map[string]string{ + "cos": "cos.%s.myqcloud.com", + "oss": "oss-%s.aliyuncs.com", + "qiniu": "s3-%s.qiniucs.com", + "google": "storage.googleapis.com", + "aws": "%s", } // New is a instantiation function to find and init a upload driver. -func New(driver, endpoint, accessKeyID, accessKeySecret, bucketName string) (Driver, error) { - if constructor, ok := supportDrivers[driver]; ok { - return constructor(endpoint, accessKeyID, accessKeySecret, bucketName) +func New(driver, region, accessKey, secretKey, bucketName string) (Driver, error) { + if _, exist := supportDrivers[driver]; !exist { + return nil, fmt.Errorf("driver[%s] not support", driver) + } + + endpoint := supportDrivers[driver] + if strings.Contains(endpoint, "%s") { + endpoint = fmt.Sprintf(endpoint, region) + } + if driver == "aws" { + endpoint = "" } - return nil, fmt.Errorf("driver[%s] not support", driver) + return NewS3Uploader(region, endpoint, accessKey, secretKey, bucketName) } diff --git a/uploader/driver_test.go b/uploader/driver_test.go index b2b2083..a44649d 100644 --- a/uploader/driver_test.go +++ b/uploader/driver_test.go @@ -4,41 +4,44 @@ import ( "io/ioutil" "log" "os" + "strings" "testing" "github.com/stretchr/testify/assert" + + "uptoc/utils" ) var driverConfigs = map[string]map[string]string{ "cos": { "bucket": "ut-uptoc-1255970412", - "endpoint": "ap-shanghai", + "region": "ap-shanghai", "access_key": os.Getenv("UPLOADER_COS_AK"), "access_secret": os.Getenv("UPLOADER_COS_SK"), }, "oss": { "bucket": "ut-uptoc", - "endpoint": "oss-cn-hangzhou.aliyuncs.com", + "region": "cn-hangzhou", "access_key": os.Getenv("UPLOADER_OSS_AK"), "access_secret": os.Getenv("UPLOADER_OSS_SK"), }, "qiniu": { "bucket": "ut-uptoc", - "endpoint": "huadong", + "region": "cn-east-1", "access_key": os.Getenv("UPLOADER_QINIU_AK"), "access_secret": os.Getenv("UPLOADER_QINIU_SK"), }, - "s3": { + "aws": { "bucket": "ut-uptoc", - "endpoint": "ap-northeast-1", + "region": "ap-northeast-1", "access_key": os.Getenv("UPLOADER_S3_AK"), "access_secret": os.Getenv("UPLOADER_S3_SK"), }, "google": { "bucket": "ut-uptoc", - "endpoint": "", - "access_key": "", - "access_secret": os.Getenv("UPLOADER_GOOGLE_SK"), + "region": "auto", + "access_key": os.Getenv("UPLOADER_STORAGE_AK"), + "access_secret": os.Getenv("UPLOADER_STORAGE_SK"), }, } @@ -58,7 +61,7 @@ func TestUploader(t *testing.T) { // test the all drivers for driver, config := range driverConfigs { log.Printf("===== driver: %s =====", driver) - uploader, err := New(driver, config["endpoint"], config["access_key"], config["access_secret"], config["bucket"]) + uploader, err := New(driver, config["region"], config["access_key"], config["access_secret"], config["bucket"]) assert.NoError(t, err) // test object upload @@ -71,6 +74,11 @@ func TestUploader(t *testing.T) { assert.NoError(t, err) assert.Equal(t, len(files), len(objects)) + // test object ETag + for _, object := range objects { + assert.Equal(t, strings.ToLower(object.ETag), utils.FileMD5(tmp+object.Key)) + } + // test object delete for object := range files { assert.NoError(t, uploader.Delete(object)) diff --git a/uploader/google.go b/uploader/google.go deleted file mode 100644 index 9d8f0b3..0000000 --- a/uploader/google.go +++ /dev/null @@ -1,91 +0,0 @@ -package uploader - -import ( - "context" - "encoding/base64" - "io" - "os" - "strings" - "time" - - "cloud.google.com/go/storage" - "google.golang.org/api/iterator" - "google.golang.org/api/option" -) - -// GoogleUploader implements the Driver base on the Storage of Google. -type GoogleUploader struct { - client *storage.BucketHandle -} - -// NewGoogleUploader returns a new google storage uploader -func NewGoogleUploader(endpoint, accessKeyID, accessKeySecret, bucketName string) (Driver, error) { - j, err := base64.StdEncoding.DecodeString(accessKeySecret) - if err != nil { - return nil, err - } - - opts := []option.ClientOption{ - //option.WithEndpoint(endpoint), - option.WithCredentialsJSON(j), - } - client, err := storage.NewClient(context.Background(), opts...) - if err != nil { - return nil, err - } - - return &GoogleUploader{ - client: client.Bucket(bucketName), - }, nil -} - -// ListObjects returns some remote objects -func (u *GoogleUploader) ListObjects() ([]Object, error) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) - defer cancel() - - objects := make([]Object, 0) - it := u.client.Objects(ctx, nil) - for { - obj, err := it.Next() - if err == iterator.Done { - break - } - if err != nil { - return nil, err - } - - objects = append(objects, Object{ - Key: obj.Name, - ETag: strings.ToLower(strings.Trim(obj.Etag, `"`)), - }) - } - - return objects, nil -} - -// Upload uploads the local file to the object -func (u *GoogleUploader) Upload(objectKey, filePath string) error { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) - defer cancel() - - bodyReader, err := os.Open(filePath) - if err != nil { - return err - } - defer bodyReader.Close() - - wc := u.client.Object(objectKey).NewWriter(ctx) - if _, err = io.Copy(wc, bodyReader); err != nil { - return err - } - - return wc.Close() -} - -// Delete deletes the object -func (u *GoogleUploader) Delete(object string) error { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*3) - defer cancel() - return u.client.Object(object).Delete(ctx) -} diff --git a/uploader/oss.go b/uploader/oss.go deleted file mode 100644 index e1a2837..0000000 --- a/uploader/oss.go +++ /dev/null @@ -1,62 +0,0 @@ -package uploader - -import ( - "strings" - - "github.com/aliyun/aliyun-oss-go-sdk/oss" -) - -// OSSUploader implements the Driver base on ali's oss. -type OSSUploader struct { - bucket *oss.Bucket -} - -// NewOSSUploader returns a new oss uploader -func NewOSSUploader(endpoint, accessKeyID, accessKeySecret, bucketName string) (Driver, error) { - ossCli, err := oss.New(endpoint, accessKeyID, accessKeySecret) - if err != nil { - return nil, err - } - - bucket, err := ossCli.Bucket(bucketName) - if err != nil { - return nil, err - } - - return &OSSUploader{ - bucket: bucket, - }, nil -} - -// ListObjects returns some remote objects -func (u *OSSUploader) ListObjects() ([]Object, error) { - marker := "" - objects := make([]Object, 0) - for { - objectsResult, err := u.bucket.ListObjects(oss.Marker(marker)) - if err != nil { - return nil, err - } - for _, obj := range objectsResult.Objects { - objects = append(objects, Object{Key: obj.Key, ETag: strings.ToLower(strings.Trim(obj.ETag, `"`))}) - } - - if objectsResult.IsTruncated { - marker = objectsResult.NextMarker - } else { - break - } - } - - return objects, nil -} - -// Upload uploads the local file to the object -func (u *OSSUploader) Upload(objectKey, filePath string) error { - return u.bucket.PutObjectFromFile(objectKey, filePath) -} - -// Delete deletes the object -func (u *OSSUploader) Delete(object string) error { - return u.bucket.DeleteObject(object) -} diff --git a/uploader/qiniu.go b/uploader/qiniu.go deleted file mode 100644 index 15030a9..0000000 --- a/uploader/qiniu.go +++ /dev/null @@ -1,88 +0,0 @@ -package uploader - -import ( - "context" - "fmt" - - "github.com/qiniu/api.v7/auth/qbox" - "github.com/qiniu/api.v7/storage" -) - -var zones = map[string]storage.Zone{ - "huadong": storage.ZoneHuadong, - "huabei": storage.ZoneHuabei, - "huanan": storage.ZoneHuanan, - "beimei": storage.ZoneBeimei, - "xinjiapo": storage.ZoneXinjiapo, -} - -// Qiniu implements the Driver base on qiuniu. -type Qiniu struct { - mac *qbox.Mac - cfg *storage.Config - bucketName string -} - -// NewQiniuUploader returns a new Qiniu uploader -func NewQiniuUploader(endpoint, accessKey, accessSecret, bucketName string) (Driver, error) { - zone, ok := zones[endpoint] - if !ok { - return nil, fmt.Errorf("endpoint %s not support", endpoint) - } - - return &Qiniu{ - mac: qbox.NewMac(accessKey, accessSecret), - cfg: &storage.Config{ - Zone: &zone, - }, - bucketName: bucketName, - }, nil -} - -// ListObjects returns some remote objects -func (u *Qiniu) ListObjects() ([]Object, error) { - limit := 1000 - prefix := "" - delimiter := "" - - //初始列举marker为空 - marker := "" - objects := make([]Object, 0) - bucket := storage.NewBucketManager(u.mac, u.cfg) - for { - entries, _, nextMarker, hashNext, err := bucket.ListFiles(u.bucketName, prefix, delimiter, marker, limit) - if err != nil { - return nil, err - } - - for _, entry := range entries { - objects = append(objects, Object{ - Key: entry.Key, - ETag: entry.Hash, - }) - } - if hashNext { - marker = nextMarker - } else { - //list end - break - } - } - - return objects, nil -} - -// Upload uploads the local file to the object -func (u *Qiniu) Upload(object, rawPath string) error { - putPolicy := storage.PutPolicy{ - Scope: u.bucketName, - } - ctx := context.Background() - upToken := putPolicy.UploadToken(u.mac) - return storage.NewFormUploader(u.cfg).PutFile(ctx, &storage.PutRet{}, upToken, object, rawPath, nil) -} - -// Delete deletes the object -func (u *Qiniu) Delete(object string) error { - return storage.NewBucketManager(u.mac, u.cfg).Delete(u.bucketName, object) -} diff --git a/uploader/s3.go b/uploader/s3.go index fb5c365..6ceb87e 100644 --- a/uploader/s3.go +++ b/uploader/s3.go @@ -17,15 +17,15 @@ type S3Uploader struct { } // NewS3Uploader returns a new s3 uploader -func NewS3Uploader(endpoint, accessKeyID, accessKeySecret, bucketName string) (Driver, error) { - cfg := aws.NewConfig().WithCredentials(credentials.NewStaticCredentials(accessKeyID, accessKeySecret, "")) +func NewS3Uploader(region, endpoint, accessKey, secretKey, bucketName string) (Driver, error) { + cfg := aws.NewConfig().WithCredentials(credentials.NewStaticCredentials(accessKey, secretKey, "")) s, err := session.NewSession(cfg) if err != nil { return nil, err } return &S3Uploader{ - client: s3.New(s, cfg.WithRegion(endpoint)), + client: s3.New(s, cfg.WithRegion(region), cfg.WithEndpoint(endpoint)), bucket: bucketName, }, nil } @@ -45,9 +45,12 @@ func (u *S3Uploader) ListObjects() ([]Object, error) { } for _, obj := range objectsResult.Contents { - objects = append(objects, Object{ + fObj := Object{ Key: aws.StringValue(obj.Key), - ETag: strings.ToLower(strings.Trim(aws.StringValue(obj.ETag), `"`))}) + ETag: strings.Trim(aws.StringValue(obj.ETag), `"`), + } + + objects = append(objects, fObj) } if aws.BoolValue(objectsResult.IsTruncated) { diff --git a/utils/utils.go b/utils/utils.go new file mode 100644 index 0000000..9c38ca1 --- /dev/null +++ b/utils/utils.go @@ -0,0 +1,23 @@ +package utils + +import ( + "crypto/md5" + "encoding/hex" + "io" + "os" +) + +func FileMD5(filepath string) string { + f, err := os.Open(filepath) + if err != nil { + return "" + } + defer f.Close() + + md5hash := md5.New() + if _, err := io.Copy(md5hash, f); err != nil { + return "" + } + + return hex.EncodeToString(md5hash.Sum(nil)[:]) +}