create etcd service

go.mod

@@ -25,7 +25,7 @@ require (
 	k8s.io/utils v0.0.0-20190607212802-c55fbcfc754a // indirect
 	yunion.io/x/jsonutils v0.0.0-20200415132054-2bf8a5e94501
 	yunion.io/x/log v0.0.0-20200313080802-57a4ce5966b3
-	yunion.io/x/onecloud v0.0.0-20200420125513-a0f1bd0c3a58
+	yunion.io/x/onecloud v0.0.0-20200422115051-5d4f7eabeeb0
 	yunion.io/x/pkg v0.0.0-20200416145704-22c189971435
 	yunion.io/x/structarg v0.0.0-20190809075558-115bed041de3
 )

go.sum

@@ -987,10 +987,8 @@ yunion.io/x/log v0.0.0-20190629062853-9f6483a7103d h1:59zrDL7Ft+hDukguJRmLr/Gdu/
 yunion.io/x/log v0.0.0-20190629062853-9f6483a7103d/go.mod h1:LC6f/4FozL0iaAbnFt2eDX9jlsyo3WiOUPm03d7+U4U=
 yunion.io/x/log v0.0.0-20200313080802-57a4ce5966b3 h1:5Wc5hkB8PtMudmHuzCyok960RuOa9I55imIGrigSdjs=
 yunion.io/x/log v0.0.0-20200313080802-57a4ce5966b3/go.mod h1:LC6f/4FozL0iaAbnFt2eDX9jlsyo3WiOUPm03d7+U4U=
-yunion.io/x/onecloud v0.0.0-20200418082822-76ee769eaf54 h1:62l6/K8Jx1YOyJs4Koag8C3UrNIdvOnf8mjvk+ZddT8=
-yunion.io/x/onecloud v0.0.0-20200418082822-76ee769eaf54/go.mod h1:l9w6FuC/yjgZTv85yJ3LoZ3KzQzaP81WjYo2VV6I7mM=
-yunion.io/x/onecloud v0.0.0-20200420125513-a0f1bd0c3a58 h1:LwKtcnT73OP1k9+BsOG8dZc9/sLhXUDEneqnYrKfaiw=
-yunion.io/x/onecloud v0.0.0-20200420125513-a0f1bd0c3a58/go.mod h1:l9w6FuC/yjgZTv85yJ3LoZ3KzQzaP81WjYo2VV6I7mM=
+yunion.io/x/onecloud v0.0.0-20200422115051-5d4f7eabeeb0 h1:5vPPcVpOydYcg6kCQbTAZbNrAlN4Cg5mIBivMeiNe7g=
+yunion.io/x/onecloud v0.0.0-20200422115051-5d4f7eabeeb0/go.mod h1:l9w6FuC/yjgZTv85yJ3LoZ3KzQzaP81WjYo2VV6I7mM=
 yunion.io/x/pkg v0.0.0-20190620104149-945c25821dbf/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
 yunion.io/x/pkg v0.0.0-20190628082551-f4033ba2ea30/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
 yunion.io/x/pkg v0.0.0-20200302034534-fdf44d54b070 h1:rKnYgtvMHKmzPEUTkyNjyKOG7wzjpUvI7fcZwLNGQXw=

pkg/apis/constants/constants.go

@@ -301,6 +301,9 @@ const (
 	EtcdDefaultDialTimeout    = 3 * time.Second
 	BusyboxImageName          = "busybox"
 	BusyboxImageVersion       = "1.28.0-glibc"
+	ServiceNameEtcd           = "etcd"
+	ServiceTypeEtcd           = ServiceNameEtcd
+	ServiceCertEtcdName       = ServiceNameEtcd
 )
 
 const (

pkg/controller/onecloud_control.go

@@ -21,17 +21,19 @@ import (
 	"sync"
 	"time"
 
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/klog"
 
 	"yunion.io/x/jsonutils"
+	"yunion.io/x/pkg/errors"
+
 	"yunion.io/x/onecloud-operator/pkg/apis/constants"
 	"yunion.io/x/onecloud-operator/pkg/apis/onecloud/v1alpha1"
 	"yunion.io/x/onecloud-operator/pkg/util/onecloud"
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/mcclient/auth"
 	"yunion.io/x/onecloud/pkg/mcclient/modules"
-	"yunion.io/x/pkg/errors"
 )
 
 var (
@@ -364,7 +366,7 @@ func (c *baseComponent) registerServiceEndpointsBySession(s *mcclient.ClientSess
 		urls[ep.Interface] = ep.GetUrl(enableSSL)
 	}
 	region := c.GetCluster().Spec.Region
-	return onecloud.RegisterServiceEndpoints(s, region, serviceName, serviceType, urls)
+	return onecloud.RegisterServiceEndpoints(s, region, serviceName, serviceType, "", urls)
 }
 
 func (c *baseComponent) RegisterServiceEndpoints(serviceName, serviceType string, eps []*endpoint, enableSSL bool) error {
@@ -433,6 +435,23 @@ func (c keystoneComponent) SystemInit() error {
 		if err := doSyncCommonConfigure(s, c.getCommonConfig()); err != nil {
 			return errors.Wrap(err, "sync common configure")
 		}
+		if !oc.Spec.Etcd.Disable {
+			var certName string
+			if oc.Spec.Etcd.EnableTls {
+				certConf, err := c.getEtcdCertificate()
+				if err != nil {
+					return errors.Wrap(err, "get etcd cert")
+				}
+				if err := doCreateEtcdCertificate(s, certConf); err != nil {
+					return errors.Wrap(err, "create etcd certificate")
+				}
+				certName = constants.ServiceCertEtcdName
+			}
+
+			if err := doCreateEtcdServiceEndpoint(s, region, c.getEtcdUrl(), certName); err != nil {
+				return errors.Wrap(err, "create etcd endpoint")
+			}
+		}
 		return nil
 	})
 }
@@ -447,6 +466,30 @@ func (c keystoneComponent) getCommonConfig() map[string]string {
 	}
 }
 
+func (c keystoneComponent) getEtcdCertificate() (*jsonutils.JSONDict, error) {
+	oc := c.GetCluster()
+	ret := jsonutils.NewDict()
+	ctl := c.baseComponent.manager.GetController()
+	secret, err := ctl.kubeCli.CoreV1().Secrets(oc.GetNamespace()).
+		Get(constants.EtcdClientSecret, metav1.GetOptions{})
+	if err != nil {
+		return nil, err
+	}
+	ret.Set("certificate", jsonutils.NewString(string(secret.Data[constants.EtcdClientCertName])))
+	ret.Set("private_key", jsonutils.NewString(string(secret.Data[constants.EtcdClientKeyName])))
+	ret.Set("ca_certificate", jsonutils.NewString(string(secret.Data[constants.EtcdClientCACertName+".crt"])))
+	return ret, nil
+}
+
+func (c keystoneComponent) getEtcdUrl() string {
+	oc := c.GetCluster()
+	scheme := "http"
+	if oc.Spec.Etcd.EnableTls {
+		scheme = "https"
+	}
+	return fmt.Sprintf("%s://%s-etcd-client.%s.svc:%d", scheme, oc.Name, oc.Namespace, constants.EtcdClientPort)
+}
+
 func shouldDoPolicyRoleInit(s *mcclient.ClientSession) (bool, error) {
 	ret, err := modules.Policies.List(s, nil)
 	if err != nil {
@@ -491,6 +534,7 @@ func doRegisterCloudMeta(s *mcclient.ClientSession, regionId string) error {
 	return onecloud.RegisterServicePublicInternalEndpoint(s, regionId,
 		constants.ServiceNameCloudmeta,
 		constants.ServiceTypeCloudmeta,
+		"",
 		constants.ServiceURLCloudmeta)
 }
 
@@ -499,6 +543,7 @@ func doRegisterTracker(s *mcclient.ClientSession, regionId string) error {
 		s, regionId,
 		constants.ServiceNameTorrentTracker,
 		constants.ServiceTypeTorrentTracker,
+		"",
 		constants.ServiceURLTorrentTracker)
 }
 
@@ -550,10 +595,23 @@ func doSyncCommonConfigure(s *mcclient.ClientSession, defaultConf map[string]str
 	return err
 }
 
+func doCreateEtcdServiceEndpoint(s *mcclient.ClientSession, regionId, endpointUrl, certName string) error {
+	return onecloud.RegisterServiceEndpointByInterfaces(
+		s, regionId, constants.ServiceNameEtcd, constants.ServiceTypeEtcd,
+		endpointUrl, certName, []string{constants.EndpointTypeInternal},
+	)
+}
+
+func doCreateEtcdCertificate(s *mcclient.ClientSession, certDetails *jsonutils.JSONDict) error {
+	_, err := onecloud.EnsureServiceCertificate(s, constants.ServiceCertEtcdName, certDetails)
+	return err
+}
+
 func doRegisterOfflineCloudMeta(s *mcclient.ClientSession, regionId string) error {
 	return onecloud.RegisterServicePublicInternalEndpoint(s, regionId,
 		constants.ServiceNameOfflineCloudmeta,
 		constants.ServiceTypeOfflineCloudmeta,
+		"",
 		constants.ServiceURLOfflineCloudmeta)
 }
 

pkg/manager/certs/manager.go

@@ -56,20 +56,22 @@ func (c *CertsManager) CreateOrUpdate(oc *v1alpha1.OnecloudCluster) error {
 		//return nil
 	}
 
-	for _, secretName := range []string{constants.EtcdServerSecret, constants.EtcdClientSecret, constants.EtcdPeerSecret} {
-		_, err := c.secretLister.Secrets(ns).Get(secretName)
-		if err != nil {
-			if !apierrors.IsNotFound(err) {
-				return err
+	if !oc.Spec.Etcd.Disable && oc.Spec.Etcd.EnableTls {
+		for _, secretName := range []string{constants.EtcdServerSecret, constants.EtcdClientSecret, constants.EtcdPeerSecret} {
+			_, err := c.secretLister.Secrets(ns).Get(secretName)
+			if err != nil {
+				if !apierrors.IsNotFound(err) {
+					return err
+				}
+				if err := c.certControl.CreateEtcdCert(oc); err != nil {
+					return errors.Wrap(err, "create cluster cert")
+				}
+				return nil
+			} else {
+				// already exists, update it
+				// TODO
+				continue
 			}
-			if err := c.certControl.CreateEtcdCert(oc); err != nil {
-				return errors.Wrap(err, "create cluster cert")
-			}
-			return nil
-		} else {
-			// already exists, update it
-			// TODO
-			continue
 		}
 	}
 	return nil

pkg/manager/component/component.go

@@ -229,13 +229,18 @@ func (m *ComponentManager) syncConfigMap(
 	if err := SetConfigMapLastAppliedConfigAnnotation(cfgMap); err != nil {
 		return err
 	}
-	oldCfgMap, _ := m.configer.Lister().ConfigMaps(oc.GetNamespace()).Get(cfgMap.GetName())
+	oldCfgMap, err := m.configer.Lister().ConfigMaps(oc.GetNamespace()).Get(cfgMap.GetName())
+	if err != nil && !errors.IsNotFound(err) {
+		return err
+	}
 	if oldCfgMap != nil {
-		if equal, err := configMapEqual(cfgMap, oldCfgMap); err != nil {
-			return err
-		} else if equal {
-			return nil
-		}
+		//if equal, err := configMapEqual(cfgMap, oldCfgMap); err != nil {
+		//	return err
+		//} else if equal {
+		//	return nil
+		//}
+		// if cfgmap exist do not update
+		return nil
 	}
 	return m.configer.CreateOrUpdateConfigMap(oc, cfgMap)
 }

pkg/manager/component/etcd.go

@@ -52,9 +52,10 @@ type etcdManager struct {
 const (
 	peerTLSDir         = "/etc/etcdtls/member/peer-tls"
 	serverTLSDir       = "/etc/etcdtls/member/server-tls"
+	operatorEtcdTLSDir = "/etc/etcdtls/operator/etcd-tls"
+
 	etcdVolumeMountDir = "/var/etcd"
 	dataDir            = etcdVolumeMountDir + "/data"
-	operatorEtcdTLSDir = "/etc/etcdtls/operator/etcd-tls"
 )
 
 var (
@@ -229,7 +230,6 @@ func (m *etcdManager) createPod(
 	pod := k8sutil.NewEtcdPod(mb, initCluster, m.getEtcdClusterPrefix(), state,
 		token, m.customEtcdSpec(), controller.GetOwnerRef(m.oc))
 	m.customPodSpec(pod, mb, state, token, initCluster)
-	// TODO: custum busybox container image
 	if m.isPodPVEnabled() {
 		pvc := k8sutil.NewEtcdPodPVC(mb, *m.oc.Spec.Etcd.Pod.PersistentVolumeClaimSpec,
 			m.oc.GetName(), m.oc.GetNamespace(), controller.GetOwnerRef(m.oc))

pkg/util/onecloud/onecloud.go

@@ -121,6 +121,17 @@ func EnsureService(s *mcclient.ClientSession, svcName, svcType string) (jsonutil
 	})
 }
 
+func EnsureServiceCertificate(s *mcclient.ClientSession, certName string, certDetails *jsonutils.JSONDict) (jsonutils.JSONObject, error) {
+	return EnsureResource(s, &modules.ServiceCertificatesV3, certName, func() (jsonutils.JSONObject, error) {
+		return CreateServiceCertificate(s, certName, certDetails)
+	})
+}
+
+func CreateServiceCertificate(s *mcclient.ClientSession, certName string, certDetails *jsonutils.JSONDict) (jsonutils.JSONObject, error) {
+	certDetails.Add(jsonutils.NewString(certName), "name")
+	return modules.ServiceCertificatesV3.Create(s, certDetails)
+}
+
 func CreateService(s *mcclient.ClientSession, svcName, svcType string) (jsonutils.JSONObject, error) {
 	params := jsonutils.NewDict()
 	params.Add(jsonutils.NewString(svcType), "type")
@@ -144,7 +155,9 @@ func IsEndpointExists(s *mcclient.ClientSession, svcId, regionId, interfaceType
 	return eps.Data[0], true, nil
 }
 
-func EnsureEndpoint(s *mcclient.ClientSession, svcId, regionId, interfaceType, url string) (jsonutils.JSONObject, error) {
+func EnsureEndpoint(
+	s *mcclient.ClientSession, svcId, regionId, interfaceType, url, serviceCert string,
+) (jsonutils.JSONObject, error) {
 	ep, exists, err := IsEndpointExists(s, svcId, regionId, interfaceType)
 	if err != nil {
 		return nil, err
@@ -156,6 +169,9 @@ func EnsureEndpoint(s *mcclient.ClientSession, svcId, regionId, interfaceType, u
 		createParams.Add(jsonutils.NewString(interfaceType), "interface")
 		createParams.Add(jsonutils.NewString(url), "url")
 		createParams.Add(jsonutils.JSONTrue, "enabled")
+		if len(serviceCert) > 0 {
+			createParams.Add(jsonutils.NewString(serviceCert), "service_certificate")
+		}
 		return modules.EndpointsV3.Create(s, createParams)
 	}
 	epId, err := ep.GetString("id")
@@ -171,6 +187,9 @@ func EnsureEndpoint(s *mcclient.ClientSession, svcId, regionId, interfaceType, u
 	updateParams := jsonutils.NewDict()
 	updateParams.Add(jsonutils.NewString(url), "url")
 	updateParams.Add(jsonutils.JSONTrue, "enabled")
+	if len(serviceCert) > 0 {
+		updateParams.Add(jsonutils.NewString(serviceCert), "service_certificate")
+	}
 	return modules.EndpointsV3.Update(s, epId, updateParams)
 }
 
@@ -387,6 +406,7 @@ func RegisterServiceEndpoints(
 	regionId string,
 	serviceName string,
 	serviceType string,
+	serviceCert string,
 	interfaceUrls map[string]string,
 ) error {
 	svc, err := EnsureService(s, serviceName, serviceType)
@@ -402,7 +422,7 @@ func RegisterServiceEndpoints(
 		tmpInf := inf
 		tmpUrl := endpointUrl
 		errgrp.Go(func() error {
-			_, err = EnsureEndpoint(s, svcId, regionId, tmpInf, tmpUrl)
+			_, err = EnsureEndpoint(s, svcId, regionId, tmpInf, tmpUrl, serviceCert)
 			if err != nil {
 				return err
 			}
@@ -418,24 +438,26 @@ func RegisterServiceEndpointByInterfaces(
 	serviceName string,
 	serviceType string,
 	endpointUrl string,
+	serviceCert string,
 	interfaces []string,
 ) error {
 	urls := make(map[string]string)
 	for _, inf := range interfaces {
 		urls[inf] = endpointUrl
 	}
-	return RegisterServiceEndpoints(s, regionId, serviceName, serviceType, urls)
+	return RegisterServiceEndpoints(s, regionId, serviceName, serviceType, serviceCert, urls)
 }
 
 func RegisterServicePublicInternalEndpoint(
 	s *mcclient.ClientSession,
 	regionId string,
 	serviceName string,
 	serviceType string,
+	serviceCert string,
 	endpointUrl string,
 ) error {
 	return RegisterServiceEndpointByInterfaces(s, regionId, serviceName, serviceType,
-		endpointUrl, []string{constants.EndpointTypeInternal, constants.EndpointTypePublic})
+		endpointUrl, serviceCert, []string{constants.EndpointTypeInternal, constants.EndpointTypePublic})
 }
 
 func ToPlaybook(

vendor/modules.txt

@@ -563,7 +563,7 @@ yunion.io/x/jsonutils
 # yunion.io/x/log v0.0.0-20200313080802-57a4ce5966b3
 yunion.io/x/log
 yunion.io/x/log/hooks
-# yunion.io/x/onecloud v0.0.0-20200418082822-76ee769eaf54
+# yunion.io/x/onecloud v0.0.0-20200422115051-5d4f7eabeeb0
 yunion.io/x/onecloud/pkg/ansibleserver/options
 yunion.io/x/onecloud/pkg/apis
 yunion.io/x/onecloud/pkg/apis/ansible