feat(registry): using etcd storage for chartmuseum (#1328)

Co-authored-by: jianzhuang <jianzhuang@tencent.com>

cmd/tke-installer/app/installer/manifests/tke-registry-api/tke-registry-api.yaml

@@ -194,6 +194,13 @@ data:
     apiVersion: registry.config.tkestack.io/v1
     kind: RegistryConfiguration
     storage:
+      etcd:
+        cafile: "/app/certs/etcd-ca.crt"
+        certfile: "/app/certs/etcd.crt"
+        keyfile: "/app/certs/etcd.key"
+        endpoints:
+        - "https://etcd.kube-system:2379"
+        prefix: "/chart_backend_bucket"
       fileSystem:
         rootDirectory: /storage
       delete:

cmd/tke-installer/app/installer/manifests/tke-registry-controller/tke-registry-controller.yaml

@@ -116,6 +116,13 @@ data:
     apiVersion: registry.config.tkestack.io/v1
     kind: RegistryConfiguration
     storage:
+      etcd:
+        cafile: "/app/certs/etcd-ca.crt"
+        certfile: "/app/certs/etcd.crt"
+        keyfile: "/app/certs/etcd.key"
+        endpoints:
+        - "https://etcd.kube-system:2379"
+        prefix: "/chart_backend_bucket"
       fileSystem:
         rootDirectory: /storage
     security:

go.mod

@@ -3,9 +3,10 @@ module tkestack.io/tke
 go 1.12
 
 replace (
-	github.com/chartmuseum/storage => github.com/choujimmy/storage v0.0.0-20200507092433-6aea2df34764
+	github.com/chartmuseum/storage => github.com/choujimmy/storage v0.5.1-0.20210412121305-660c0e91489b
 	github.com/containerd/containerd => github.com/containerd/containerd v1.4.3
 	github.com/deislabs/oras => github.com/deislabs/oras v0.8.0
+	go.etcd.io/etcd => go.etcd.io/etcd v0.5.0-alpha.5.0.20200819165624-17cef6e3e9d5
 	google.golang.org/grpc => google.golang.org/grpc v1.26.0
 	k8s.io/api => k8s.io/api v0.19.7
 	k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.19.7

go.sum

@@ -220,8 +220,8 @@ github.com/chartmuseum/helm-push v0.9.0/go.mod h1:zu/IQlPG84gTLsKDmGF0wnK5WiPer1
 github.com/checkpoint-restore/go-criu/v4 v4.0.2/go.mod h1:xUQBLp4RLc5zJtWY++yjOoMoB5lihDt7fai+75m+rGw=
 github.com/cheekybits/genny v0.0.0-20170328200008-9127e812e1e9/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ=
 github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ=
-github.com/choujimmy/storage v0.0.0-20200507092433-6aea2df34764 h1:Ne0n7pvPyXsfYH1mHjDCiqhVcmpthIQSgblJy/xpGK4=
-github.com/choujimmy/storage v0.0.0-20200507092433-6aea2df34764/go.mod h1:n6BVxAk/c5IJniwjfjjBUdZ7JXyvTHbAugy27J5hdYg=
+github.com/choujimmy/storage v0.5.1-0.20210412121305-660c0e91489b h1:CtmmTUQ5q0FO4pLjkkLY1mbkFnMDyQDU0r0CIhun8BM=
+github.com/choujimmy/storage v0.5.1-0.20210412121305-660c0e91489b/go.mod h1:UtDnuHQVC5s03bSaHJHoszGc/jdxYTFaSkx8ZZ4X5og=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
@@ -529,6 +529,8 @@ github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4er
 github.com/golang/groupcache v0.0.0-20191027212112-611e8accdfc9/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7 h1:5ZkaAPbicIKTF2I64qf5Fh8Aa83Q/dnOafMYV0OMwjA=
 github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
@@ -1209,7 +1211,6 @@ go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0=
 go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
-go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg=
 go.etcd.io/etcd v0.5.0-alpha.5.0.20200819165624-17cef6e3e9d5 h1:Gqga3zA9tdAcfqobUGjSoCob5L3f8Dt5EuOp3ihNZko=
 go.etcd.io/etcd v0.5.0-alpha.5.0.20200819165624-17cef6e3e9d5/go.mod h1:skWido08r9w6Lq/w70DO5XYIKMu4QFu1+4VsqLQuJy8=
 go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
@@ -1432,9 +1433,8 @@ golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxb
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1 h1:NusfzzA6yGQ+ua51ck7E3omNUX/JuqbFSaRGqU8CcLI=
-golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180810170437-e96c4e24768d/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -1527,6 +1527,7 @@ google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvx
 google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
 google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
 google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
 google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY=
 google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
 google.golang.org/grpc v1.26.0 h1:2dTRdpdFEEhJYQD8EMLB61nnrzSCTbG38PhqdhvOltg=

pkg/registry/apis/config/types.go

@@ -34,9 +34,9 @@ type RegistryConfiguration struct {
 	Redis         *Redis
 	DefaultTenant string
 	// +optional
-	DomainSuffix string
+	DomainSuffix  string
 	HarborEnabled bool
-	HarborCAFile string
+	HarborCAFile  string
 }
 
 type Storage struct {
@@ -47,8 +47,18 @@ type Storage struct {
 	// +optional
 	S3 *S3Storage
 	// +optional
+	Etcd *EtcdStorage
+	// +optional
 	Delete *Delete
 }
+type EtcdStorage struct {
+	CAFile    string
+	CertFile  string
+	KeyFile   string
+	EndPoints []string
+	// +optional
+	Prefix string
+}
 
 type FileSystemStorage struct {
 	RootDirectory string

pkg/registry/apis/config/v1/types.go

@@ -47,9 +47,20 @@ type Storage struct {
 	// +optional
 	S3 *S3Storage `json:"s3,omitempty" yaml:"s3,omitempty"`
 	// +optional
+	Etcd *EtcdStorage `json:"etcd,omitempty" yaml:"etcd,omitempty"`
+	// +optional
 	Delete *Delete `json:"delete,omitempty" yaml:"delete,omitempty"`
 }
 
+type EtcdStorage struct {
+	CAFile    string   `json:"cafile" yaml:"cafile"`
+	CertFile  string   `json:"certfile" yaml:"certfile"`
+	KeyFile   string   `json:"keyfile" yaml:"keyfile"`
+	EndPoints []string `json:"endpoints" yaml:"endpoints"`
+	// +optional
+	Prefix string `json:"prefix,omitempty" yaml:"prefix,omitempty"`
+}
+
 type FileSystemStorage struct {
 	RootDirectory string `json:"rootDirectory" yaml:"rootDirectory"`
 	// +optional

pkg/registry/apis/config/validation/validation.go

@@ -31,6 +31,24 @@ func ValidateRegistryConfiguration(rc *registryconfig.RegistryConfiguration) err
 	storageCount := 0
 	storageFld := field.NewPath("storage")
 
+	if rc.Storage.Etcd != nil {
+		storageCount++
+		subFld := storageFld.Child("etcd")
+
+		if rc.Storage.Etcd.CAFile == "" {
+			allErrors = append(allErrors, field.Required(subFld.Child("cafile"), "must be specify"))
+		}
+		if rc.Storage.Etcd.CertFile == "" {
+			allErrors = append(allErrors, field.Required(subFld.Child("certfile"), "must be specify"))
+		}
+		if rc.Storage.Etcd.KeyFile == "" {
+			allErrors = append(allErrors, field.Required(subFld.Child("keyfile"), "must be specify"))
+		}
+		if len(rc.Storage.Etcd.EndPoints) == 0 {
+			allErrors = append(allErrors, field.Required(subFld.Child("endpoints"), "must be specify"))
+		}
+	}
+
 	if rc.Storage.S3 != nil {
 		storageCount++
 		subFld := storageFld.Child("s3")

pkg/registry/chartmuseum/serveroptions/serveroptions.go

@@ -84,9 +84,14 @@ func buildStorageConfiguration(registryConfig *registryconfig.RegistryConfigurat
 	var backend storage.Backend
 	var err error
 	storageCfg := &registryConfig.Storage
-	if storageCfg.FileSystem != nil {
+	if storageCfg.Etcd != nil {
+		log.Info("Using etcd storage")
+		backend, err = buildETCDStorageConfiguration(registryConfig.Storage.Etcd)
+	} else if storageCfg.FileSystem != nil {
+		log.Info("Using filesystem storage")
 		backend = storage.Backend(storage.NewLocalFilesystemBackend(storageCfg.FileSystem.RootDirectory))
 	} else if storageCfg.S3 != nil {
+		log.Info("Using s3 storage")
 		backend, err = buildS3StorageConfiguration(registryConfig.Storage.S3)
 	}
 
@@ -100,6 +105,16 @@ func buildStorageConfiguration(registryConfig *registryconfig.RegistryConfigurat
 	return backend, nil
 }
 
+func buildETCDStorageConfiguration(cfg *registryconfig.EtcdStorage) (storage.Backend, error) {
+	return storage.NewEtcdCSBackend(
+		strings.Join(cfg.EndPoints, ","),
+		cfg.CAFile,
+		cfg.CertFile,
+		cfg.KeyFile,
+		cfg.Prefix,
+	), nil
+}
+
 func buildS3StorageConfiguration(cfg *registryconfig.S3Storage) (storage.Backend, error) {
 	awsConfig := aws.NewConfig()
 	sess, err := session.NewSession()