Skip to content

Commit

Permalink
Merge pull request #167 from alimy/pr-huawei-obs
Browse files Browse the repository at this point in the history
support Huawei Cloud OBS as OSS service
  • Loading branch information
alimy committed Jul 23, 2022
2 parents 500adeb + 4232099 commit 878fd1b
Show file tree
Hide file tree
Showing 10 changed files with 172 additions and 17 deletions.
5 changes: 4 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
.PHONY: all build run test clean fmt help
.PHONY: all build run test clean fmt pre-commit help

TARGET = paopao-ce
ifeq ($(OS),Windows_NT)
Expand Down Expand Up @@ -82,6 +82,9 @@ fmt:
test:
@go test ./...

pre-commit: fmt
go mod tidy

help:
@echo "make: make"
@echo "make run: start api server"
Expand Down
7 changes: 4 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -306,9 +306,10 @@ release/paopao-ce --no-default-features --features sqlite3,localoss,loggerfile,r
* 数据库: MySQL/Sqlite3/PostgreSQL
`Gorm` + `MySQL`/`Sqlite3`/`PostgreSQL` 使用[gorm](https://github.com/go-gorm/gorm)作为数据库的ORM,默认使用 `Grom` + `MySQL`组合(目前状态:稳定,默认,推荐使用);
`Sqlx` + `MySQL`/`PostgreSQL` 使用[sqlx](https://github.com/jmoiron/sqlx)作为数据库的ORM(目前状态:WIP);
* 对象存储: AliOSS/COS/MinIO/LocalOSS
`AliOSS` 阿里云对象存储服务;
`COS` 腾讯云对象存储服务
* 对象存储: AliOSS/COS/HuaweiOBS/MinIO/LocalOSS
`AliOSS` 阿里云对象存储服务;
`COS` 腾讯云对象存储服务;
`HuaweiOBS` 华为云对象存储服务;
`MinIO` [MinIO](https://github.com/minio/minio)对象存储服务;
`LocalOSS` 提供使用本地目录文件作为对象存储的功能,仅用于开发调试环境;
* 缓存: Redis/SimpleCacheIndex/BigCacheIndex
Expand Down
6 changes: 6 additions & 0 deletions config.yaml.sample
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,12 @@ COS: # 腾讯云COS存储配置
Region: ap-shanghai
Bucket: demo-1888888888
Domain:
HuaweiOBS: # 华为云OBS存储配置
AccessKey:
SecretKey:
Endpoint:
Bucket: paopao
Domain:
MinIO: # MinIO 存储配置
AccessKey: Q3AM3UQ867SPQQA43P2F
SecretKey: zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG
Expand Down
3 changes: 2 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ require (
github.com/gofrs/uuid v4.0.0+incompatible
github.com/golang-migrate/migrate/v4 v4.15.2
github.com/google/go-cmp v0.5.7 // indirect
github.com/huaweicloud/huaweicloud-sdk-go-obs v3.21.12+incompatible
github.com/json-iterator/go v1.1.12
github.com/meilisearch/meilisearch-go v0.19.1
github.com/minio/minio-go/v7 v7.0.27
Expand All @@ -30,7 +31,7 @@ require (
github.com/sirupsen/logrus v1.8.1
github.com/smartwalle/alipay/v3 v3.1.7
github.com/spf13/viper v1.10.1
github.com/tencentyun/cos-go-sdk-v5 v0.7.35 // indirect
github.com/tencentyun/cos-go-sdk-v5 v0.7.35
github.com/ugorji/go v1.2.7 // indirect
github.com/yinheli/mahonia v0.0.0-20131226213531-0eef680515cc
golang.org/x/crypto v0.0.0-20220307211146-efcb8507fb70 // indirect
Expand Down
3 changes: 2 additions & 1 deletion go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -823,6 +823,8 @@ github.com/hashicorp/serf v0.9.6/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpT
github.com/holiman/bloomfilter/v2 v2.0.3/go.mod h1:zpoh+gs7qcpqrHr3dB55AMiJwo0iURXE7ZOP9L9hSkA=
github.com/holiman/uint256 v1.2.0/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/huaweicloud/huaweicloud-sdk-go-obs v3.21.12+incompatible h1:tANYIteuFrosKbRYUk1Yo/OGJjbt4x3OVg211Qc60M0=
github.com/huaweicloud/huaweicloud-sdk-go-obs v3.21.12+incompatible/go.mod h1:l7VUhRbTKCzdOacdT4oWCwATKyvZqUOlOqr0Ous3k4s=
github.com/huin/goupnp v1.0.2/go.mod h1:0dxJBVBHqTMjIUMkESDTNgOOx/Mw5wYIfyFmdzSamkM=
github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o=
github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho=
Expand Down Expand Up @@ -1104,7 +1106,6 @@ github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:F
github.com/mitchellh/mapstructure v0.0.0-20180220230111-00c29f56e238/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/mitchellh/mapstructure v1.4.3 h1:OVowDSCllw/YjdLkam3/sm7wEtOy59d8ndGgCcyj8cs=
github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
Expand Down
4 changes: 4 additions & 0 deletions internal/conf/conf.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ var (
MeiliSetting *MeiliSettingS
AliOSSSetting *AliOSSSettingS
COSSetting *COSSettingS
HuaweiOBSSetting *HuaweiOBSSettingS
MinIOSetting *MinIOSettingS
S3Setting *S3SettingS
LocalOSSSetting *LocalOSSSettingS
Expand Down Expand Up @@ -73,6 +74,7 @@ func setupSetting(suite []string, noDefault bool) error {
"JWT": &JWTSetting,
"AliOSS": &AliOSSSetting,
"COS": &COSSetting,
"HuaweiOBS": &HuaweiOBSSetting,
"MinIO": &MinIOSetting,
"LocalOSS": &LocalOSSSetting,
"S3": &S3Setting,
Expand Down Expand Up @@ -121,6 +123,8 @@ func GetOssDomain() string {
return uri + AliOSSSetting.Domain + "/"
} else if CfgIf("COS") {
return uri + COSSetting.Domain + "/"
} else if CfgIf("HuaweiOBS") {
return uri + HuaweiOBSSetting.Domain + "/"
} else if CfgIf("MinIO") {
if !MinIOSetting.Secure {
uri = "http://"
Expand Down
10 changes: 10 additions & 0 deletions internal/conf/settting.go
Original file line number Diff line number Diff line change
Expand Up @@ -159,13 +159,23 @@ type AliOSSSettingS struct {
Bucket string
Domain string
}

type COSSettingS struct {
SecretID string
SecretKey string
Region string
Bucket string
Domain string
}

type HuaweiOBSSettingS struct {
AccessKey string
SecretKey string
Endpoint string
Bucket string
Domain string
}

type LocalOSSSettingS struct {
SavePath string
Secure bool
Expand Down
14 changes: 8 additions & 6 deletions internal/dao/dao.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,20 +43,22 @@ func ObjectStorageService() core.ObjectStorageService {
onceOss.Do(func() {
var v core.VersionInfo
if conf.CfgIf("AliOSS") {
oss, v = storage.NewAliossService()
oss, v = storage.MustAliossService()
} else if conf.CfgIf("COS") {
oss, v = storage.NewCosServent()
oss, v = storage.NewCosService()
} else if conf.CfgIf("HuaweiOBS") {
oss, v = storage.MustHuaweiobsService()
} else if conf.CfgIf("MinIO") {
oss, v = storage.NewMinioService()
oss, v = storage.MustMinioService()
} else if conf.CfgIf("S3") {
oss, v = storage.NewS3Service()
oss, v = storage.MustS3Service()
logrus.Infof("use S3 as object storage by version %s", v.Version())
return
} else if conf.CfgIf("LocalOSS") {
oss, v = storage.NewLocalossService()
oss, v = storage.MustLocalossService()
} else {
// default use AliOSS as object storage service
oss, v = storage.NewAliossService()
oss, v = storage.MustAliossService()
logrus.Infof("use default AliOSS as object storage by version %s", v.Version())
return
}
Expand Down
112 changes: 112 additions & 0 deletions internal/dao/storage/huaweiobs.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,112 @@
package storage

import (
"io"
"net/url"
"strings"

"github.com/Masterminds/semver/v3"
"github.com/huaweicloud/huaweicloud-sdk-go-obs/obs"
"github.com/rocboss/paopao-ce/internal/core"
"github.com/sirupsen/logrus"
)

var (
_ core.ObjectStorageService = (*huaweiobsServant)(nil)
_ core.VersionInfo = (*huaweiobsServant)(nil)
)

type huaweiobsServant struct {
client *obs.ObsClient
bucket string
domain string
}

func (s *huaweiobsServant) Name() string {
return "HuaweiOBS"
}

func (s *huaweiobsServant) Version() *semver.Version {
return semver.MustParse("v0.1.0")
}

func (s *huaweiobsServant) PutObject(objectKey string, reader io.Reader, objectSize int64, contentType string) (string, error) {
input := &obs.PutObjectInput{}
input.Bucket, input.Key, input.Body = s.bucket, objectKey, reader
input.ContentType, input.ContentLength = contentType, objectSize
_, err := s.client.PutObject(input)
if err != nil {
return "", err
}
return s.domain + objectKey, nil
}

func (s *huaweiobsServant) DeleteObject(objectKey string) error {
_, err := s.client.DeleteObject(&obs.DeleteObjectInput{
Bucket: s.bucket,
Key: objectKey,
})
return err
}

func (s *huaweiobsServant) DeleteObjects(objectKeys []string) error {
input := &obs.DeleteObjectsInput{
Bucket: s.bucket,
Objects: make([]obs.ObjectToDelete, 0, len(objectKeys)),
}

for _, key := range objectKeys {
input.Objects = append(input.Objects, obs.ObjectToDelete{Key: key})
}

_, err := s.client.DeleteObjects(input)
return err
}

func (s *huaweiobsServant) IsObjectExist(objectKey string) (bool, error) {
input := &obs.GetObjectMetadataInput{
Bucket: s.bucket,
Key: objectKey,
}
if _, err := s.client.GetObjectMetadata(input); err != nil {
return false, err
}
return true, nil
}

func (s *huaweiobsServant) SignURL(objectKey string, expiredInSec int64) (string, error) {
input := &obs.CreateSignedUrlInput{
Method: obs.HttpMethodGet,
Bucket: s.bucket,
Key: objectKey,
Expires: int(expiredInSec),
}
out, err := s.client.CreateSignedUrl(input)
if err != nil {
logrus.Errorf("huaweiobsServant.SignURL err: %v", err)
return "", err
}

ur, err := url.Parse(out.SignedUrl)
if err != nil {
logrus.Errorf("url.Parse err: %v", err)
return "", err
}

epath, err := url.PathUnescape(ur.Path)
if err != nil {
logrus.Errorf("url.PathUnescape err: %v", err)
return "", err
}

ur.Path, ur.RawPath = epath, epath
return ur.String(), nil
}

func (s *huaweiobsServant) ObjectURL(objetKey string) string {
return s.domain + objetKey
}

func (s *huaweiobsServant) ObjectKey(objectUrl string) string {
return strings.Replace(objectUrl, s.domain, "", -1)
}
25 changes: 20 additions & 5 deletions internal/dao/storage/storage.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"path/filepath"

"github.com/aliyun/aliyun-oss-go-sdk/oss"
"github.com/huaweicloud/huaweicloud-sdk-go-obs/obs"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
"github.com/rocboss/paopao-ce/internal/conf"
Expand All @@ -15,7 +16,7 @@ import (
"github.com/tencentyun/cos-go-sdk-v5"
)

func NewAliossService() (core.ObjectStorageService, core.VersionInfo) {
func MustAliossService() (core.ObjectStorageService, core.VersionInfo) {
client, err := oss.New(conf.AliOSSSetting.Endpoint, conf.AliOSSSetting.AccessKeyID, conf.AliOSSSetting.AccessKeySecret)
if err != nil {
logrus.Fatalf("alioss.New err: %v", err)
Expand All @@ -33,7 +34,7 @@ func NewAliossService() (core.ObjectStorageService, core.VersionInfo) {
return obj, obj
}

func NewCosServent() (core.ObjectStorageService, core.VersionInfo) {
func NewCosService() (core.ObjectStorageService, core.VersionInfo) {
u, _ := url.Parse(fmt.Sprintf("https://%s.cos.%s.myqcloud.com", conf.COSSetting.Bucket, conf.COSSetting.Region))
su, _ := url.Parse(fmt.Sprintf("https://cos.%s.myqcloud.com", conf.COSSetting.Region))

Expand All @@ -51,7 +52,21 @@ func NewCosServent() (core.ObjectStorageService, core.VersionInfo) {
return obj, obj
}

func NewLocalossService() (core.ObjectStorageService, core.VersionInfo) {
func MustHuaweiobsService() (core.ObjectStorageService, core.VersionInfo) {
s := conf.HuaweiOBSSetting
client, err := obs.New(s.AccessKey, s.SecretKey, s.Endpoint)
if err != nil {
logrus.Fatalf("create huawei obs client failed: %s", err)
}
obj := &huaweiobsServant{
client: client,
bucket: s.Bucket,
domain: conf.GetOssDomain(),
}
return obj, obj
}

func MustLocalossService() (core.ObjectStorageService, core.VersionInfo) {
savePath, err := filepath.Abs(conf.LocalOSSSetting.SavePath)
if err != nil {
logrus.Fatalf("get localOSS save path err: %v", err)
Expand All @@ -64,7 +79,7 @@ func NewLocalossService() (core.ObjectStorageService, core.VersionInfo) {
return obj, obj
}

func NewMinioService() (core.ObjectStorageService, core.VersionInfo) {
func MustMinioService() (core.ObjectStorageService, core.VersionInfo) {
// Initialize minio client object.
client, err := minio.New(conf.MinIOSetting.Endpoint, &minio.Options{
Creds: credentials.NewStaticV4(conf.MinIOSetting.AccessKey, conf.MinIOSetting.SecretKey, ""),
Expand All @@ -82,7 +97,7 @@ func NewMinioService() (core.ObjectStorageService, core.VersionInfo) {
return ms, ms
}

func NewS3Service() (core.ObjectStorageService, core.VersionInfo) {
func MustS3Service() (core.ObjectStorageService, core.VersionInfo) {
// Initialize s3 client object use minio-go.
client, err := minio.New(conf.S3Setting.Endpoint, &minio.Options{
Creds: credentials.NewStaticV4(conf.S3Setting.AccessKey, conf.S3Setting.SecretKey, ""),
Expand Down

0 comments on commit 878fd1b

Please sign in to comment.