Skip to content

Commit

Permalink
etcd tls endpoint
Browse files Browse the repository at this point in the history
  • Loading branch information
wanyaoqi committed Apr 23, 2020
1 parent e25a2f7 commit 69f02e3
Show file tree
Hide file tree
Showing 10 changed files with 266 additions and 46 deletions.
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ require (
k8s.io/utils v0.0.0-20190607212802-c55fbcfc754a // indirect
yunion.io/x/jsonutils v0.0.0-20200415132054-2bf8a5e94501
yunion.io/x/log v0.0.0-20200313080802-57a4ce5966b3
yunion.io/x/onecloud v0.0.0-20200418082822-76ee769eaf54
yunion.io/x/onecloud v0.0.0-20200420125513-a0f1bd0c3a58
yunion.io/x/pkg v0.0.0-20200416145704-22c189971435
yunion.io/x/structarg v0.0.0-20190809075558-115bed041de3
)
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -989,6 +989,8 @@ yunion.io/x/log v0.0.0-20200313080802-57a4ce5966b3 h1:5Wc5hkB8PtMudmHuzCyok960Ru
yunion.io/x/log v0.0.0-20200313080802-57a4ce5966b3/go.mod h1:LC6f/4FozL0iaAbnFt2eDX9jlsyo3WiOUPm03d7+U4U=
yunion.io/x/onecloud v0.0.0-20200418082822-76ee769eaf54 h1:62l6/K8Jx1YOyJs4Koag8C3UrNIdvOnf8mjvk+ZddT8=
yunion.io/x/onecloud v0.0.0-20200418082822-76ee769eaf54/go.mod h1:l9w6FuC/yjgZTv85yJ3LoZ3KzQzaP81WjYo2VV6I7mM=
yunion.io/x/onecloud v0.0.0-20200420125513-a0f1bd0c3a58 h1:LwKtcnT73OP1k9+BsOG8dZc9/sLhXUDEneqnYrKfaiw=
yunion.io/x/onecloud v0.0.0-20200420125513-a0f1bd0c3a58/go.mod h1:l9w6FuC/yjgZTv85yJ3LoZ3KzQzaP81WjYo2VV6I7mM=
yunion.io/x/pkg v0.0.0-20190620104149-945c25821dbf/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
yunion.io/x/pkg v0.0.0-20190628082551-f4033ba2ea30/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
yunion.io/x/pkg v0.0.0-20200302034534-fdf44d54b070 h1:rKnYgtvMHKmzPEUTkyNjyKOG7wzjpUvI7fcZwLNGQXw=
Expand Down
13 changes: 10 additions & 3 deletions pkg/apis/constants/constants.go
Original file line number Diff line number Diff line change
Expand Up @@ -361,13 +361,20 @@ var (

Localhost = "localhost"

EtcdServerCACertName = "server-ca.crt"
EtcdServerSecret = "etcd-server"
EtcdClientSecret = "etcd-client"
EtcdPeerSecret = "etcd-peer"

EtcdServerName = "server"
EtcdServerCACertName = "server-ca"
EtcdServerCertName = "server.crt"
EtcdServerKeyName = "server.key"
EtcdClientCACertName = "etcd-client-ca.crt"
EtcdClientName = "etcd-client"
EtcdClientCACertName = "etcd-client-ca"
EtcdClientCertName = "etcd-client.crt"
EtcdClientKeyName = "etcd-client.key"
EtcdPeerCACertName = "peer-ca.crt"
EtcdPeerName = "peer"
EtcdPeerCACertName = "peer-ca"
EtcdPeerCertName = "peer.crt"
EtcdPeerKeyName = "peer.key"
)
2 changes: 1 addition & 1 deletion pkg/controller/cluster/onecloud_cluster_control.go
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,7 @@ func (occ *defaultClusterControl) updateOnecloudCluster(oc *v1alpha1.OnecloudClu
if err := components.Etcd().Sync(oc); err != nil {
return err
}

for _, component := range []manager.Manager{
components.Keystone(),
components.Region(),
Expand Down Expand Up @@ -139,6 +140,5 @@ func (occ *defaultClusterControl) updateOnecloudCluster(oc *v1alpha1.OnecloudClu
if err := grp.Wait(); err != nil {
return err
}

return nil
}
1 change: 0 additions & 1 deletion pkg/controller/cluster/onecloud_cluster_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -194,7 +194,6 @@ func (c *Controller) worker() {
// invoked concurrently with the same key.
func (c *Controller) processNextWorkItem() bool {
key, quit := c.queue.Get()
// log.Errorf("queue get KEY is %s ......", key.(string))
if quit {
return false
}
Expand Down
1 change: 1 addition & 0 deletions pkg/controller/onecloud_cert.go
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ type OnecloudCertControlInterface interface {

// GetCertSecret return certs secret
// GetCertSecret(oc *v1alpha1.OnecloudCluster) (*corev1.Secret, error)
CreateEtcdCert(oc *v1alpha1.OnecloudCluster) error
}

type realOnecloudCertControl struct {
Expand Down
219 changes: 219 additions & 0 deletions pkg/controller/onecloud_etcd_cert.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,219 @@
package controller

import (
"crypto/x509"
"fmt"
"net"

"yunion.io/x/log"

corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
certutil "k8s.io/client-go/util/cert"

"yunion.io/x/onecloud-operator/pkg/apis/constants"
"yunion.io/x/onecloud-operator/pkg/apis/onecloud/v1alpha1"
)

func NewEtcdClusterCACert() *OnecloudCert {
return &OnecloudCert{
Name: "ca",
LongName: "self-signed onecloud cA to provision identities for etcd components",
BaseName: constants.EtcdServerCACertName,
CAName: "ca",
config: certutil.Config{
CommonName: "onecloud",
},
}
}

func NewEtcdServerCert(caName string, serviceName string, certName string) *OnecloudCert {
return &OnecloudCert{
Name: serviceName,
LongName: fmt.Sprintf("certificate for serving the %s service", serviceName),
BaseName: serviceName,
CAName: caName,
config: certutil.Config{
CommonName: serviceName,
Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
},
configMutators: []configMutatorsFunc{
makeAltNamesMutator(
getServerAltNames,
serviceName,
certName,
),
},
}
}

func NewEtcdClientCert(caName string, serviceName string, certName string) *OnecloudCert {
return &OnecloudCert{
Name: serviceName,
LongName: fmt.Sprintf("certificate for serving the %s service", serviceName),
BaseName: serviceName,
CAName: caName,
config: certutil.Config{
CommonName: serviceName,
Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
},
configMutators: []configMutatorsFunc{
makeAltNamesMutator(
getClientAltNames,
serviceName,
certName,
),
},
}
}

func NewEtcdPeerCert(caName string, serviceName string, certName string) *OnecloudCert {
return &OnecloudCert{
Name: serviceName,
LongName: fmt.Sprintf("certificate for serving the %s service", serviceName),
BaseName: serviceName,
CAName: caName,
config: certutil.Config{
CommonName: serviceName,
Usages: []x509.ExtKeyUsage{
x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth,
},
},
configMutators: []configMutatorsFunc{
makeAltNamesMutator(
getPeerAltNames,
serviceName,
certName,
),
},
}
}

func getServerAltNames(
oc *v1alpha1.OnecloudCluster, serviceName string, certName string,
) (*certutil.AltNames, error) {
ns := oc.GetNamespace()
altNames := &certutil.AltNames{
DNSNames: []string{
fmt.Sprintf("*.%s-etcd.%s.svc", oc.Name, ns),
fmt.Sprintf("%s-etcd-client.%s.svc", oc.Name, ns),
constants.Localhost,
},
IPs: []net.IP{
net.ParseIP("127.0.0.1"),
},
}

return altNames, nil
}

func getPeerAltNames(oc *v1alpha1.OnecloudCluster, serviceName string, certName string) (*certutil.AltNames, error) {
ns := oc.GetNamespace()
altNames := &certutil.AltNames{
DNSNames: []string{
fmt.Sprintf("*.%s-etcd.%s.svc", oc.Name, ns),
fmt.Sprintf("*.%s-etcd.%s.svc.cluster.local", oc.Name, ns),
constants.Localhost,
},
}

return altNames, nil
}

func getClientAltNames(oc *v1alpha1.OnecloudCluster, serviceName string, certName string) (*certutil.AltNames, error) {
altNames := &certutil.AltNames{
DNSNames: []string{
"",
},
}

return altNames, nil
}

func (c *realOnecloudCertControl) CreateEtcdCert(oc *v1alpha1.OnecloudCluster) error {
for _, secretName := range []string{constants.EtcdServerSecret, constants.EtcdClientSecret, constants.EtcdPeerSecret} {
err := c.kubeCli.CoreV1().Secrets(oc.GetNamespace()).Delete(secretName, new(metav1.DeleteOptions))
if err != nil {
log.Errorf("Delete secret %s failed: %s", secretName, err)
}
}
caCert := NewEtcdClusterCACert()
config, err := caCert.GetConfig(oc)
if err != nil {
return err
}
cert, key, err := NewCACertAndKey(config)
if err != nil {
return err
}

// server
store := newCertsStore()
if err := store.WriteCert(constants.EtcdServerCACertName, cert); err != nil {
return err
}
svcCerts := NewEtcdServerCert(caCert.BaseName, constants.EtcdServerName, constants.EtcdServerCertName)
svcCert, svcKey, err := svcCerts.CreateFromCA(oc, cert, key)
if err != nil {
return err
}
if err := store.WriteCertAndKey(svcCerts.BaseName, svcCert, svcKey); err != nil {
return err
}
certSecret := newEtcdSecretFromStore(oc, constants.EtcdServerSecret, store)
_, err = c.kubeCli.CoreV1().Secrets(oc.GetNamespace()).Create(certSecret)
if err != nil {
return err
}

// client
store = newCertsStore()
if err := store.WriteCert(constants.EtcdClientCACertName, cert); err != nil {
return err
}
cliCerts := NewEtcdClientCert(caCert.BaseName, constants.EtcdClientName, constants.EtcdClientCertName)
cliCert, cliKey, err := cliCerts.CreateFromCA(oc, cert, key)
if err != nil {
return err
}
if err := store.WriteCertAndKey(cliCerts.BaseName, cliCert, cliKey); err != nil {
return err
}
certSecret = newEtcdSecretFromStore(oc, constants.EtcdClientSecret, store)
_, err = c.kubeCli.CoreV1().Secrets(oc.GetNamespace()).Create(certSecret)
if err != nil {
return err
}

// peer
store = newCertsStore()
if err := store.WriteCert(constants.EtcdPeerCACertName, cert); err != nil {
return err
}
peerCerts := NewEtcdPeerCert(caCert.BaseName, constants.EtcdPeerName, constants.EtcdPeerCertName)
peerCert, peerKey, err := peerCerts.CreateFromCA(oc, cert, key)
if err != nil {
return err
}
if err := store.WriteCertAndKey(peerCerts.BaseName, peerCert, peerKey); err != nil {
return err
}
certSecret = newEtcdSecretFromStore(oc, constants.EtcdPeerSecret, store)
_, err = c.kubeCli.CoreV1().Secrets(oc.GetNamespace()).Create(certSecret)
if err != nil {
return err
}

return nil
}

func newEtcdSecretFromStore(oc *v1alpha1.OnecloudCluster, name string, store certsStore) *corev1.Secret {
return &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: oc.GetNamespace(),
OwnerReferences: []metav1.OwnerReference{GetOwnerRef(oc)},
},
Data: store,
}
}
22 changes: 20 additions & 2 deletions pkg/manager/certs/manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import (
"github.com/pkg/errors"
apierrors "k8s.io/apimachinery/pkg/api/errors"
corelisters "k8s.io/client-go/listers/core/v1"
"yunion.io/x/onecloud-operator/pkg/apis/constants"
"yunion.io/x/onecloud-operator/pkg/apis/onecloud/v1alpha1"
"yunion.io/x/onecloud-operator/pkg/controller"
)
Expand Down Expand Up @@ -50,9 +51,26 @@ func (c *CertsManager) CreateOrUpdate(oc *v1alpha1.OnecloudCluster) error {
}
return nil
} else {
// TODO
// already exists, update it
// TODO
return nil
//return nil
}

for _, secretName := range []string{constants.EtcdServerSecret, constants.EtcdClientSecret, constants.EtcdPeerSecret} {
_, err := c.secretLister.Secrets(ns).Get(secretName)
if err != nil {
if !apierrors.IsNotFound(err) {
return err
}
if err := c.certControl.CreateEtcdCert(oc); err != nil {
return errors.Wrap(err, "create cluster cert")
}
return nil
} else {
// already exists, update it
// TODO
continue
}
}
return nil
}

0 comments on commit 69f02e3

Please sign in to comment.