refactor: use the s3 replace the cloud platform sdk

.gitignore

@@ -14,4 +14,5 @@
 # Dependency directories (remove the comment below to include it)
 vendor/
 build/
-.coverprofile
+.coverprofile
+.driverenvs.sh

core/engine.go

@@ -1,15 +1,13 @@
 package core
 
 import (
-	"crypto/md5"
-	"encoding/hex"
-	"io"
 	"log"
 	"os"
 	"path/filepath"
 	"strings"
 
 	"uptoc/uploader"
+	"uptoc/utils"
 )
 
 // Engine is the core to finish the logic
@@ -92,21 +90,6 @@ func (e *Engine) Sync() error {
 	return nil
 }
 
-func fileMD5(filepath string) string {
-	f, err := os.Open(filepath)
-	if err != nil {
-		return ""
-	}
-	defer f.Close()
-
-	md5hash := md5.New()
-	if _, err := io.Copy(md5hash, f); err != nil {
-		return ""
-	}
-
-	return hex.EncodeToString(md5hash.Sum(nil)[:])
-}
-
 func objectExist(object uploader.Object, objects []uploader.Object) bool {
 	for _, obj := range objects {
 		if obj.Key == object.Key {
@@ -118,7 +101,7 @@ func objectExist(object uploader.Object, objects []uploader.Object) bool {
 
 func objectNotMatch(object uploader.Object, objects []uploader.Object) bool {
 	for _, obj := range objects {
-		if obj.Key == object.Key && obj.ETag != object.ETag {
+		if obj.Key == object.Key && strings.ToLower(obj.ETag) != object.ETag {
 			return true
 		}
 	}
@@ -134,7 +117,7 @@ func loadLocalObjects(dirPath string) ([]uploader.Object, error) {
 
 		localObjects = append(localObjects, uploader.Object{
 			Key:      strings.TrimPrefix(filePath, dirPath),
-			ETag:     fileMD5(filePath),
+			ETag:     utils.FileMD5(filePath),
 			FilePath: filePath,
 		})
 		return nil

scripts/driverenv.sh

@@ -0,0 +1,21 @@
+#!/bin/sh -l
+
+# aliyun oss
+export UPLOADER_OSS_AK=LTAI4***********nL7YoV
+export UPLOADER_OSS_SK=PFGVw**********************72Rd09u
+
+# tencent cos
+export UPLOADER_COS_SK=GH0ScC***********OWY2Cn8H
+export UPLOADER_COS_AK=AKIDTh**********************DWYKn1h5
+
+# qiniu
+export UPLOADER_QINIU_AK=dpFVS1rXn**********************9zSPwLs
+export UPLOADER_QINIU_SK=AY4DV*********************************F9JfeFoyE
+
+# aws s3
+export UPLOADER_S3_AK=AKIA***********AA
+export UPLOADER_S3_SK=/UGCFwk**********************bZmx9n
+
+# google storage
+export UPLOADER_STORAGE_AK=GOOG***********VPM7Q
+export UPLOADER_STORAGE_SK=bHw**********************wUIMnCb8

uploader/driver.go

@@ -2,6 +2,7 @@ package uploader
 
 import (
 	"fmt"
+	"strings"
 )
 
 const (
@@ -28,22 +29,27 @@ type Driver interface {
 	Delete(object string) error
 }
 
-// Constructor defines the upload driver constructor used by some engine
-type Constructor func(endpoint, accessKeyID, accessKeySecret, bucketName string) (Driver, error)
-
-var supportDrivers = map[string]Constructor{
-	"oss":    NewOSSUploader,
-	"cos":    NewCOSUploader,
-	"qiniu":  NewQiniuUploader,
-	"s3":     NewS3Uploader,
-	"google": NewGoogleUploader,
+var supportDrivers = map[string]string{
+	"cos":    "cos.%s.myqcloud.com",
+	"oss":    "oss-%s.aliyuncs.com",
+	"qiniu":  "s3-%s.qiniucs.com",
+	"google": "storage.googleapis.com",
+	"aws":    "%s",
 }
 
 // New is a instantiation function to find and init a upload driver.
-func New(driver, endpoint, accessKeyID, accessKeySecret, bucketName string) (Driver, error) {
-	if constructor, ok := supportDrivers[driver]; ok {
-		return constructor(endpoint, accessKeyID, accessKeySecret, bucketName)
+func New(driver, region, accessKey, secretKey, bucketName string) (Driver, error) {
+	if _, exist := supportDrivers[driver]; !exist {
+		return nil, fmt.Errorf("driver[%s] not support", driver)
+	}
+
+	endpoint := supportDrivers[driver]
+	if strings.Contains(endpoint, "%s") {
+		endpoint = fmt.Sprintf(endpoint, region)
+	}
+	if driver == "aws" {
+		endpoint = ""
 	}
 
-	return nil, fmt.Errorf("driver[%s] not support", driver)
+	return NewS3Uploader(region, endpoint, accessKey, secretKey, bucketName)
 }

uploader/driver_test.go

@@ -4,41 +4,44 @@ import (
 	"io/ioutil"
 	"log"
 	"os"
+	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+
+	"uptoc/utils"
 )
 
 var driverConfigs = map[string]map[string]string{
 	"cos": {
 		"bucket":        "ut-uptoc-1255970412",
-		"endpoint":      "ap-shanghai",
+		"region":        "ap-shanghai",
 		"access_key":    os.Getenv("UPLOADER_COS_AK"),
 		"access_secret": os.Getenv("UPLOADER_COS_SK"),
 	},
 	"oss": {
 		"bucket":        "ut-uptoc",
-		"endpoint":      "oss-cn-hangzhou.aliyuncs.com",
+		"region":        "cn-hangzhou",
 		"access_key":    os.Getenv("UPLOADER_OSS_AK"),
 		"access_secret": os.Getenv("UPLOADER_OSS_SK"),
 	},
 	"qiniu": {
 		"bucket":        "ut-uptoc",
-		"endpoint":      "huadong",
+		"region":        "cn-east-1",
 		"access_key":    os.Getenv("UPLOADER_QINIU_AK"),
 		"access_secret": os.Getenv("UPLOADER_QINIU_SK"),
 	},
-	"s3": {
+	"aws": {
 		"bucket":        "ut-uptoc",
-		"endpoint":      "ap-northeast-1",
+		"region":        "ap-northeast-1",
 		"access_key":    os.Getenv("UPLOADER_S3_AK"),
 		"access_secret": os.Getenv("UPLOADER_S3_SK"),
 	},
 	"google": {
 		"bucket":        "ut-uptoc",
-		"endpoint":      "",
-		"access_key":    "",
-		"access_secret": os.Getenv("UPLOADER_GOOGLE_SK"),
+		"region":        "auto",
+		"access_key":    os.Getenv("UPLOADER_STORAGE_AK"),
+		"access_secret": os.Getenv("UPLOADER_STORAGE_SK"),
 	},
 }
 
@@ -58,7 +61,7 @@ func TestUploader(t *testing.T) {
 	// test the all drivers
 	for driver, config := range driverConfigs {
 		log.Printf("===== driver: %s =====", driver)
-		uploader, err := New(driver, config["endpoint"], config["access_key"], config["access_secret"], config["bucket"])
+		uploader, err := New(driver, config["region"], config["access_key"], config["access_secret"], config["bucket"])
 		assert.NoError(t, err)
 
 		// test object upload
@@ -71,6 +74,11 @@ func TestUploader(t *testing.T) {
 		assert.NoError(t, err)
 		assert.Equal(t, len(files), len(objects))
 
+		// test object ETag
+		for _, object := range objects {
+			assert.Equal(t, strings.ToLower(object.ETag), utils.FileMD5(tmp+object.Key))
+		}
+
 		// test object delete
 		for object := range files {
 			assert.NoError(t, uploader.Delete(object))