support Huawei Cloud OBS as OSS service

Makefile

@@ -1,4 +1,4 @@
-.PHONY: all build run test clean fmt help
+.PHONY: all build run test clean fmt pre-commit help
 
 TARGET = paopao-ce
 ifeq ($(OS),Windows_NT)
@@ -82,6 +82,9 @@ fmt:
 test:
 	@go test ./...
 
+pre-commit: fmt
+	go mod tidy
+
 help:
 	@echo "make: make"
 	@echo "make run: start api server"

README.md

@@ -306,9 +306,10 @@ release/paopao-ce --no-default-features --features sqlite3,localoss,loggerfile,r
 * 数据库: MySQL/Sqlite3/PostgreSQL   
   `Gorm` + `MySQL`/`Sqlite3`/`PostgreSQL` 使用[gorm](https://github.com/go-gorm/gorm)作为数据库的ORM，默认使用 `Grom` + `MySQL`组合(目前状态：稳定，默认，推荐使用)；    
   `Sqlx` + `MySQL`/`PostgreSQL` 使用[sqlx](https://github.com/jmoiron/sqlx)作为数据库的ORM(目前状态：WIP)；      
-* 对象存储: AliOSS/COS/MinIO/LocalOSS    
-  `AliOSS` 阿里云对象存储服务；  
-  `COS` 腾讯云对象存储服务
+* 对象存储: AliOSS/COS/HuaweiOBS/MinIO/LocalOSS      
+  `AliOSS` 阿里云对象存储服务；    
+  `COS` 腾讯云对象存储服务；  
+  `HuaweiOBS` 华为云对象存储服务；  
   `MinIO` [MinIO](https://github.com/minio/minio)对象存储服务；  
   `LocalOSS` 提供使用本地目录文件作为对象存储的功能，仅用于开发调试环境；  
 * 缓存: Redis/SimpleCacheIndex/BigCacheIndex     

config.yaml.sample

@@ -85,6 +85,12 @@ COS: # 腾讯云COS存储配置
   Region: ap-shanghai
   Bucket: demo-1888888888
   Domain: 
+HuaweiOBS: # 华为云OBS存储配置
+  AccessKey: 
+  SecretKey: 
+  Endpoint: 
+  Bucket: paopao
+  Domain: 
 MinIO: # MinIO 存储配置
   AccessKey: Q3AM3UQ867SPQQA43P2F
   SecretKey: zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG

go.mod

@@ -22,6 +22,7 @@ require (
 	github.com/gofrs/uuid v4.0.0+incompatible
 	github.com/golang-migrate/migrate/v4 v4.15.2
 	github.com/google/go-cmp v0.5.7 // indirect
+	github.com/huaweicloud/huaweicloud-sdk-go-obs v3.21.12+incompatible
 	github.com/json-iterator/go v1.1.12
 	github.com/meilisearch/meilisearch-go v0.19.1
 	github.com/minio/minio-go/v7 v7.0.27
@@ -30,7 +31,7 @@ require (
 	github.com/sirupsen/logrus v1.8.1
 	github.com/smartwalle/alipay/v3 v3.1.7
 	github.com/spf13/viper v1.10.1
-	github.com/tencentyun/cos-go-sdk-v5 v0.7.35 // indirect
+	github.com/tencentyun/cos-go-sdk-v5 v0.7.35
 	github.com/ugorji/go v1.2.7 // indirect
 	github.com/yinheli/mahonia v0.0.0-20131226213531-0eef680515cc
 	golang.org/x/crypto v0.0.0-20220307211146-efcb8507fb70 // indirect

go.sum

@@ -823,6 +823,8 @@ github.com/hashicorp/serf v0.9.6/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpT
 github.com/holiman/bloomfilter/v2 v2.0.3/go.mod h1:zpoh+gs7qcpqrHr3dB55AMiJwo0iURXE7ZOP9L9hSkA=
 github.com/holiman/uint256 v1.2.0/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/huaweicloud/huaweicloud-sdk-go-obs v3.21.12+incompatible h1:tANYIteuFrosKbRYUk1Yo/OGJjbt4x3OVg211Qc60M0=
+github.com/huaweicloud/huaweicloud-sdk-go-obs v3.21.12+incompatible/go.mod h1:l7VUhRbTKCzdOacdT4oWCwATKyvZqUOlOqr0Ous3k4s=
 github.com/huin/goupnp v1.0.2/go.mod h1:0dxJBVBHqTMjIUMkESDTNgOOx/Mw5wYIfyFmdzSamkM=
 github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o=
 github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho=
@@ -1104,7 +1106,6 @@ github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:F
 github.com/mitchellh/mapstructure v0.0.0-20180220230111-00c29f56e238/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
-github.com/mitchellh/mapstructure v1.4.3 h1:OVowDSCllw/YjdLkam3/sm7wEtOy59d8ndGgCcyj8cs=
 github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
 github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=

internal/conf/conf.go

@@ -30,6 +30,7 @@ var (
 	MeiliSetting            *MeiliSettingS
 	AliOSSSetting           *AliOSSSettingS
 	COSSetting              *COSSettingS
+	HuaweiOBSSetting        *HuaweiOBSSettingS
 	MinIOSetting            *MinIOSettingS
 	S3Setting               *S3SettingS
 	LocalOSSSetting         *LocalOSSSettingS
@@ -73,6 +74,7 @@ func setupSetting(suite []string, noDefault bool) error {
 		"JWT":              &JWTSetting,
 		"AliOSS":           &AliOSSSetting,
 		"COS":              &COSSetting,
+		"HuaweiOBS":        &HuaweiOBSSetting,
 		"MinIO":            &MinIOSetting,
 		"LocalOSS":         &LocalOSSSetting,
 		"S3":               &S3Setting,
@@ -121,6 +123,8 @@ func GetOssDomain() string {
 		return uri + AliOSSSetting.Domain + "/"
 	} else if CfgIf("COS") {
 		return uri + COSSetting.Domain + "/"
+	} else if CfgIf("HuaweiOBS") {
+		return uri + HuaweiOBSSetting.Domain + "/"
 	} else if CfgIf("MinIO") {
 		if !MinIOSetting.Secure {
 			uri = "http://"

internal/conf/settting.go

@@ -159,13 +159,23 @@ type AliOSSSettingS struct {
 	Bucket          string
 	Domain          string
 }
+
 type COSSettingS struct {
 	SecretID  string
 	SecretKey string
 	Region    string
 	Bucket    string
 	Domain    string
 }
+
+type HuaweiOBSSettingS struct {
+	AccessKey string
+	SecretKey string
+	Endpoint  string
+	Bucket    string
+	Domain    string
+}
+
 type LocalOSSSettingS struct {
 	SavePath string
 	Secure   bool

internal/dao/dao.go

@@ -43,20 +43,22 @@ func ObjectStorageService() core.ObjectStorageService {
 	onceOss.Do(func() {
 		var v core.VersionInfo
 		if conf.CfgIf("AliOSS") {
-			oss, v = storage.NewAliossService()
+			oss, v = storage.MustAliossService()
 		} else if conf.CfgIf("COS") {
-			oss, v = storage.NewCosServent()
+			oss, v = storage.NewCosService()
+		} else if conf.CfgIf("HuaweiOBS") {
+			oss, v = storage.MustHuaweiobsService()
 		} else if conf.CfgIf("MinIO") {
-			oss, v = storage.NewMinioService()
+			oss, v = storage.MustMinioService()
 		} else if conf.CfgIf("S3") {
-			oss, v = storage.NewS3Service()
+			oss, v = storage.MustS3Service()
 			logrus.Infof("use S3 as object storage by version %s", v.Version())
 			return
 		} else if conf.CfgIf("LocalOSS") {
-			oss, v = storage.NewLocalossService()
+			oss, v = storage.MustLocalossService()
 		} else {
 			// default use AliOSS as object storage service
-			oss, v = storage.NewAliossService()
+			oss, v = storage.MustAliossService()
 			logrus.Infof("use default AliOSS as object storage by version %s", v.Version())
 			return
 		}

internal/dao/storage/huaweiobs.go

@@ -0,0 +1,112 @@
+package storage
+
+import (
+	"io"
+	"net/url"
+	"strings"
+
+	"github.com/Masterminds/semver/v3"
+	"github.com/huaweicloud/huaweicloud-sdk-go-obs/obs"
+	"github.com/rocboss/paopao-ce/internal/core"
+	"github.com/sirupsen/logrus"
+)
+
+var (
+	_ core.ObjectStorageService = (*huaweiobsServant)(nil)
+	_ core.VersionInfo          = (*huaweiobsServant)(nil)
+)
+
+type huaweiobsServant struct {
+	client *obs.ObsClient
+	bucket string
+	domain string
+}
+
+func (s *huaweiobsServant) Name() string {
+	return "HuaweiOBS"
+}
+
+func (s *huaweiobsServant) Version() *semver.Version {
+	return semver.MustParse("v0.1.0")
+}
+
+func (s *huaweiobsServant) PutObject(objectKey string, reader io.Reader, objectSize int64, contentType string) (string, error) {
+	input := &obs.PutObjectInput{}
+	input.Bucket, input.Key, input.Body = s.bucket, objectKey, reader
+	input.ContentType, input.ContentLength = contentType, objectSize
+	_, err := s.client.PutObject(input)
+	if err != nil {
+		return "", err
+	}
+	return s.domain + objectKey, nil
+}
+
+func (s *huaweiobsServant) DeleteObject(objectKey string) error {
+	_, err := s.client.DeleteObject(&obs.DeleteObjectInput{
+		Bucket: s.bucket,
+		Key:    objectKey,
+	})
+	return err
+}
+
+func (s *huaweiobsServant) DeleteObjects(objectKeys []string) error {
+	input := &obs.DeleteObjectsInput{
+		Bucket:  s.bucket,
+		Objects: make([]obs.ObjectToDelete, 0, len(objectKeys)),
+	}
+
+	for _, key := range objectKeys {
+		input.Objects = append(input.Objects, obs.ObjectToDelete{Key: key})
+	}
+
+	_, err := s.client.DeleteObjects(input)
+	return err
+}
+
+func (s *huaweiobsServant) IsObjectExist(objectKey string) (bool, error) {
+	input := &obs.GetObjectMetadataInput{
+		Bucket: s.bucket,
+		Key:    objectKey,
+	}
+	if _, err := s.client.GetObjectMetadata(input); err != nil {
+		return false, err
+	}
+	return true, nil
+}
+
+func (s *huaweiobsServant) SignURL(objectKey string, expiredInSec int64) (string, error) {
+	input := &obs.CreateSignedUrlInput{
+		Method:  obs.HttpMethodGet,
+		Bucket:  s.bucket,
+		Key:     objectKey,
+		Expires: int(expiredInSec),
+	}
+	out, err := s.client.CreateSignedUrl(input)
+	if err != nil {
+		logrus.Errorf("huaweiobsServant.SignURL err: %v", err)
+		return "", err
+	}
+
+	ur, err := url.Parse(out.SignedUrl)
+	if err != nil {
+		logrus.Errorf("url.Parse err: %v", err)
+		return "", err
+	}
+
+	epath, err := url.PathUnescape(ur.Path)
+	if err != nil {
+		logrus.Errorf("url.PathUnescape err: %v", err)
+		return "", err
+	}
+
+	ur.Path, ur.RawPath = epath, epath
+	return ur.String(), nil
+}
+
+func (s *huaweiobsServant) ObjectURL(objetKey string) string {
+	return s.domain + objetKey
+}
+
+func (s *huaweiobsServant) ObjectKey(objectUrl string) string {
+	return strings.Replace(objectUrl, s.domain, "", -1)
+}

internal/dao/storage/storage.go

@@ -7,6 +7,7 @@ import (
 	"path/filepath"
 
 	"github.com/aliyun/aliyun-oss-go-sdk/oss"
+	"github.com/huaweicloud/huaweicloud-sdk-go-obs/obs"
 	"github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/credentials"
 	"github.com/rocboss/paopao-ce/internal/conf"
@@ -15,7 +16,7 @@ import (
 	"github.com/tencentyun/cos-go-sdk-v5"
 )
 
-func NewAliossService() (core.ObjectStorageService, core.VersionInfo) {
+func MustAliossService() (core.ObjectStorageService, core.VersionInfo) {
 	client, err := oss.New(conf.AliOSSSetting.Endpoint, conf.AliOSSSetting.AccessKeyID, conf.AliOSSSetting.AccessKeySecret)
 	if err != nil {
 		logrus.Fatalf("alioss.New err: %v", err)
@@ -33,7 +34,7 @@ func NewAliossService() (core.ObjectStorageService, core.VersionInfo) {
 	return obj, obj
 }
 
-func NewCosServent() (core.ObjectStorageService, core.VersionInfo) {
+func NewCosService() (core.ObjectStorageService, core.VersionInfo) {
 	u, _ := url.Parse(fmt.Sprintf("https://%s.cos.%s.myqcloud.com", conf.COSSetting.Bucket, conf.COSSetting.Region))
 	su, _ := url.Parse(fmt.Sprintf("https://cos.%s.myqcloud.com", conf.COSSetting.Region))
 
@@ -51,7 +52,21 @@ func NewCosServent() (core.ObjectStorageService, core.VersionInfo) {
 	return obj, obj
 }
 
-func NewLocalossService() (core.ObjectStorageService, core.VersionInfo) {
+func MustHuaweiobsService() (core.ObjectStorageService, core.VersionInfo) {
+	s := conf.HuaweiOBSSetting
+	client, err := obs.New(s.AccessKey, s.SecretKey, s.Endpoint)
+	if err != nil {
+		logrus.Fatalf("create huawei obs client failed: %s", err)
+	}
+	obj := &huaweiobsServant{
+		client: client,
+		bucket: s.Bucket,
+		domain: conf.GetOssDomain(),
+	}
+	return obj, obj
+}
+
+func MustLocalossService() (core.ObjectStorageService, core.VersionInfo) {
 	savePath, err := filepath.Abs(conf.LocalOSSSetting.SavePath)
 	if err != nil {
 		logrus.Fatalf("get localOSS save path err: %v", err)
@@ -64,7 +79,7 @@ func NewLocalossService() (core.ObjectStorageService, core.VersionInfo) {
 	return obj, obj
 }
 
-func NewMinioService() (core.ObjectStorageService, core.VersionInfo) {
+func MustMinioService() (core.ObjectStorageService, core.VersionInfo) {
 	// Initialize minio client object.
 	client, err := minio.New(conf.MinIOSetting.Endpoint, &minio.Options{
 		Creds:  credentials.NewStaticV4(conf.MinIOSetting.AccessKey, conf.MinIOSetting.SecretKey, ""),
@@ -82,7 +97,7 @@ func NewMinioService() (core.ObjectStorageService, core.VersionInfo) {
 	return ms, ms
 }
 
-func NewS3Service() (core.ObjectStorageService, core.VersionInfo) {
+func MustS3Service() (core.ObjectStorageService, core.VersionInfo) {
 	// Initialize s3 client object use minio-go.
 	client, err := minio.New(conf.S3Setting.Endpoint, &minio.Options{
 		Creds:  credentials.NewStaticV4(conf.S3Setting.AccessKey, conf.S3Setting.SecretKey, ""),