From e807b1619a68b6290f81f6c140f114fb1d67829e Mon Sep 17 00:00:00 2001 From: Kirill Garbar Date: Sun, 5 May 2024 21:30:55 +0000 Subject: [PATCH] add auth enabler --- .gitignore | 1 + api/v1alpha1/aux_functions.go | 25 ++ api/v1alpha1/etcdcluster_types.go | 16 +- api/v1alpha1/etcdcluster_webhook.go | 9 + charts/etcd-operator/crds/etcd-cluster.yaml | 26 +- .../templates/workload/deployment.yml | 6 + .../crd/bases/etcd.aenix.io_etcdclusters.yaml | 26 +- config/manager/manager.yaml | 6 + internal/controller/etcdcluster_controller.go | 268 ++++++++++++++++++ internal/controller/factory/statefulset.go | 16 +- 10 files changed, 384 insertions(+), 15 deletions(-) create mode 100644 api/v1alpha1/aux_functions.go diff --git a/.gitignore b/.gitignore index b447f47a..ab2a770c 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,7 @@ vendor # editor and IDE paraphernalia .idea +.vscode *.swp *.swo *~ diff --git a/api/v1alpha1/aux_functions.go b/api/v1alpha1/aux_functions.go new file mode 100644 index 00000000..b1b61c62 --- /dev/null +++ b/api/v1alpha1/aux_functions.go @@ -0,0 +1,25 @@ +package v1alpha1 + +func IsClientSecurityEnabled(c *EtcdCluster) bool { + clientSecurityEnabled := false + if c.Spec.Security != nil && c.Spec.Security.TLS.ClientSecret != "" { + clientSecurityEnabled = true + } + return clientSecurityEnabled +} + +func IsServerSecurityEnabled(c *EtcdCluster) bool { + serverSecurityEnabled := false + if c.Spec.Security != nil && c.Spec.Security.TLS.ServerSecret != "" { + serverSecurityEnabled = true + } + return serverSecurityEnabled +} + +func IsServerCADefined(c *EtcdCluster) bool { + serverCADefined := false + if c.Spec.Security != nil && c.Spec.Security.TLS.ServerTrustedCASecret != "" { + serverCADefined = true + } + return serverCADefined +} diff --git a/api/v1alpha1/etcdcluster_types.go b/api/v1alpha1/etcdcluster_types.go index 205861ff..eb54cd10 100644 --- a/api/v1alpha1/etcdcluster_types.go +++ b/api/v1alpha1/etcdcluster_types.go @@ -174,24 +174,36 @@ type SecuritySpec struct { // Section for user-managed tls certificates // +optional TLS TLSSpec `json:"tls,omitempty"` + // Section to enable etcd auth + EnableAuth bool `json:"enableAuth,omitempty"` } // TLSSpec defines user-managed certificates names. type TLSSpec struct { - // Trusted CA certificate secret to secure peer-to-peer communication between etcd nodes. It is expected to have tls.crt field in the secret. + // Trusted CA certificate secret to secure peer-to-peer communication between etcd nodes. It is expected to have ca.crt field in the secret. + // This secret must be created in the namespace with etcdCluster CR. // +optional PeerTrustedCASecret string `json:"peerTrustedCASecret,omitempty"` // Certificate secret to secure peer-to-peer communication between etcd nodes. It is expected to have tls.crt and tls.key fields in the secret. + // This secret must be created in the namespace with etcdCluster CR. // +optional PeerSecret string `json:"peerSecret,omitempty"` + // Trusted CA for etcd server certificates for client-server communication. Is necessary to set trust between operator and etcd. + // It is expected to have ca.crt field in the secret. If it is not specified, then insecure communication will be used. + // This secret must be created in the namespace with etcdCluster CR. + // +optional + ServerTrustedCASecret string `json:"serverTrustedCASecret,omitempty"` // Server certificate secret to secure client-server communication. Is provided to the client who connects to etcd by client port (2379 by default). // It is expected to have tls.crt and tls.key fields in the secret. + // This secret must be created in the namespace with etcdCluster CR. // +optional ServerSecret string `json:"serverSecret,omitempty"` - // Trusted CA for client certificates that are provided by client to etcd. It is expected to have tls.crt field in the secret. + // Trusted CA for client certificates that are provided by client to etcd. It is expected to have ca.crt field in the secret. + // This secret must be created in the namespace with etcdCluster CR. // +optional ClientTrustedCASecret string `json:"clientTrustedCASecret,omitempty"` // Client certificate for etcd-operator to do maintenance. It is expected to have tls.crt and tls.key fields in the secret. + // This secret must be created in the namespace with etcdCluster CR. // +optional ClientSecret string `json:"clientSecret,omitempty"` } diff --git a/api/v1alpha1/etcdcluster_webhook.go b/api/v1alpha1/etcdcluster_webhook.go index 210efb02..18c7971b 100644 --- a/api/v1alpha1/etcdcluster_webhook.go +++ b/api/v1alpha1/etcdcluster_webhook.go @@ -281,6 +281,15 @@ func (r *EtcdCluster) validateSecurity() field.ErrorList { ) } + if security.EnableAuth && (security.TLS.ClientSecret == "" || security.TLS.ServerSecret == "") { + + allErrors = append(allErrors, field.Invalid( + field.NewPath("spec", "security"), + security.TLS, + "if auth is enabled, client secret and server secret must be provided"), + ) + } + if len(allErrors) > 0 { return allErrors } diff --git a/charts/etcd-operator/crds/etcd-cluster.yaml b/charts/etcd-operator/crds/etcd-cluster.yaml index 9120e191..8c79e5fd 100644 --- a/charts/etcd-operator/crds/etcd-cluster.yaml +++ b/charts/etcd-operator/crds/etcd-cluster.yaml @@ -202,25 +202,43 @@ spec: security: description: Security describes security settings of etcd (authentication, certificates, rbac) properties: + enableAuth: + description: Section to enable etcd auth + type: boolean tls: description: Section for user-managed tls certificates properties: clientSecret: - description: Client certificate for etcd-operator to do maintenance. It is expected to have tls.crt and tls.key fields in the secret. + description: |- + Client certificate for etcd-operator to do maintenance. It is expected to have tls.crt and tls.key fields in the secret. + This secret must be created in the namespace with etcd-operator. type: string clientTrustedCASecret: - description: Trusted CA for client certificates that are provided by client to etcd. It is expected to have tls.crt field in the secret. + description: |- + Trusted CA for client certificates that are provided by client to etcd. It is expected to have tls.crt field in the secret. + This secret must be created in the namespace with etcdCluster CR. type: string peerSecret: - description: Certificate secret to secure peer-to-peer communication between etcd nodes. It is expected to have tls.crt and tls.key fields in the secret. + description: |- + Certificate secret to secure peer-to-peer communication between etcd nodes. It is expected to have tls.crt and tls.key fields in the secret. + This secret must be created in the namespace with etcdCluster CR. type: string peerTrustedCASecret: - description: Trusted CA certificate secret to secure peer-to-peer communication between etcd nodes. It is expected to have tls.crt field in the secret. + description: |- + Trusted CA certificate secret to secure peer-to-peer communication between etcd nodes. It is expected to have tls.crt field in the secret. + This secret must be created in the namespace with etcdCluster CR. type: string serverSecret: description: |- Server certificate secret to secure client-server communication. Is provided to the client who connects to etcd by client port (2379 by default). It is expected to have tls.crt and tls.key fields in the secret. + This secret must be created in the namespace with etcdCluster CR. + type: string + serverTrustedCASecret: + description: |- + Trusted CA for etcd server certificates for client-server communication. Is necessary to set trust between operator and etcd. + It is expected to have tls.crt field in the secret. If it is not specified, then insecure communication will be used. + This secret must be created in the namespace with etcd-operator. type: string type: object type: object diff --git a/charts/etcd-operator/templates/workload/deployment.yml b/charts/etcd-operator/templates/workload/deployment.yml index ece5bae8..17848ada 100644 --- a/charts/etcd-operator/templates/workload/deployment.yml +++ b/charts/etcd-operator/templates/workload/deployment.yml @@ -58,6 +58,12 @@ spec: - configMapRef: name: {{ include "etcd-operator.fullname" . }}-env {{- end }} + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace volumeMounts: - mountPath: /tmp/k8s-webhook-server/serving-certs name: cert diff --git a/config/crd/bases/etcd.aenix.io_etcdclusters.yaml b/config/crd/bases/etcd.aenix.io_etcdclusters.yaml index 78ede0b4..b4134dbf 100644 --- a/config/crd/bases/etcd.aenix.io_etcdclusters.yaml +++ b/config/crd/bases/etcd.aenix.io_etcdclusters.yaml @@ -192,25 +192,43 @@ spec: security: description: Security describes security settings of etcd (authentication, certificates, rbac) properties: + enableAuth: + description: Section to enable etcd auth + type: boolean tls: description: Section for user-managed tls certificates properties: clientSecret: - description: Client certificate for etcd-operator to do maintenance. It is expected to have tls.crt and tls.key fields in the secret. + description: |- + Client certificate for etcd-operator to do maintenance. It is expected to have tls.crt and tls.key fields in the secret. + This secret must be created in the namespace with etcdCluster CR. type: string clientTrustedCASecret: - description: Trusted CA for client certificates that are provided by client to etcd. It is expected to have tls.crt field in the secret. + description: |- + Trusted CA for client certificates that are provided by client to etcd. It is expected to have ca.crt field in the secret. + This secret must be created in the namespace with etcdCluster CR. type: string peerSecret: - description: Certificate secret to secure peer-to-peer communication between etcd nodes. It is expected to have tls.crt and tls.key fields in the secret. + description: |- + Certificate secret to secure peer-to-peer communication between etcd nodes. It is expected to have tls.crt and tls.key fields in the secret. + This secret must be created in the namespace with etcdCluster CR. type: string peerTrustedCASecret: - description: Trusted CA certificate secret to secure peer-to-peer communication between etcd nodes. It is expected to have tls.crt field in the secret. + description: |- + Trusted CA certificate secret to secure peer-to-peer communication between etcd nodes. It is expected to have ca.crt field in the secret. + This secret must be created in the namespace with etcdCluster CR. type: string serverSecret: description: |- Server certificate secret to secure client-server communication. Is provided to the client who connects to etcd by client port (2379 by default). It is expected to have tls.crt and tls.key fields in the secret. + This secret must be created in the namespace with etcdCluster CR. + type: string + serverTrustedCASecret: + description: |- + Trusted CA for etcd server certificates for client-server communication. Is necessary to set trust between operator and etcd. + It is expected to have ca.crt field in the secret. If it is not specified, then insecure communication will be used. + This secret must be created in the namespace with etcdCluster CR. type: string type: object type: object diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index b62c4319..d657a5e9 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -99,5 +99,11 @@ spec: requests: cpu: 10m memory: 64Mi + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace serviceAccountName: controller-manager terminationGracePeriodSeconds: 10 diff --git a/internal/controller/etcdcluster_controller.go b/internal/controller/etcdcluster_controller.go index d3834494..6ae33c34 100644 --- a/internal/controller/etcdcluster_controller.go +++ b/internal/controller/etcdcluster_controller.go @@ -18,8 +18,16 @@ package controller import ( "context" + "crypto/tls" + "crypto/x509" + "encoding/pem" goerrors "errors" "fmt" + "os" + "slices" + "strconv" + "strings" + "time" policyv1 "k8s.io/api/policy/v1" "sigs.k8s.io/controller-runtime/pkg/log" @@ -37,6 +45,8 @@ import ( etcdaenixiov1alpha1 "github.com/aenix-io/etcd-operator/api/v1alpha1" "github.com/aenix-io/etcd-operator/internal/controller/factory" + + clientv3 "go.etcd.io/etcd/client/v3" ) // EtcdClusterReconciler reconciles a EtcdCluster object @@ -97,6 +107,13 @@ func (r *EtcdClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) return r.updateStatusOnErr(ctx, instance, fmt.Errorf("cannot check Cluster readiness: %w", err)) } + if clusterReady { + reconcileResult, err := r.configureAuth(ctx, instance) + if err != nil { + return reconcileResult, err + } + } + // set cluster readiness condition existingCondition := factory.GetCondition(instance, etcdaenixiov1alpha1.EtcdConditionReady) if existingCondition != nil && @@ -127,20 +144,36 @@ func (r *EtcdClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) // ensureClusterObjects creates or updates all objects owned by cluster CR func (r *EtcdClusterReconciler) ensureClusterObjects( ctx context.Context, cluster *etcdaenixiov1alpha1.EtcdCluster) error { + logger := log.FromContext(ctx) if err := factory.CreateOrUpdateClusterStateConfigMap(ctx, cluster, r.Client); err != nil { + logger.Error(err, "reconcile cluster state configmap failed") return err + } else { + logger.Info("cluster state configmap reconciled") } if err := factory.CreateOrUpdateHeadlessService(ctx, cluster, r.Client); err != nil { + logger.Error(err, "reconcile headless service failed") return err + } else { + logger.Info("headless service reconciled") } if err := factory.CreateOrUpdateStatefulSet(ctx, cluster, r.Client); err != nil { + logger.Error(err, "reconcile statefulset failed") return err + } else { + logger.Info("statefulset reconciled") } if err := factory.CreateOrUpdateClientService(ctx, cluster, r.Client); err != nil { + logger.Error(err, "reconcile client service failed") return err + } else { + logger.Info("client service reconciled") } if err := factory.CreateOrUpdatePdb(ctx, cluster, r.Client); err != nil { + logger.Error(err, "reconcile pdb failed") return err + } else { + logger.Info("pdb reconciled") } return nil } @@ -192,3 +225,238 @@ func (r *EtcdClusterReconciler) SetupWithManager(mgr ctrl.Manager) error { Owns(&policyv1.PodDisruptionBudget{}). Complete(r) } + +func (r *EtcdClusterReconciler) configureAuth(ctx context.Context, cluster *etcdaenixiov1alpha1.EtcdCluster) (ctrl.Result, error) { + logger := log.FromContext(ctx) + var err error + + cli, reconcileResult, err := r.getEtcdClient(ctx, cluster) + if err != nil { + return reconcileResult, err + } + + reconcileResult, err = testMemberList(ctx, cli) + if err != nil { + return reconcileResult, err + } + + auth := clientv3.NewAuth(cli) + + if cluster.Spec.Security != nil && cluster.Spec.Security.EnableAuth { + + // Create root role + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + _, err = auth.RoleGet(ctx, "root") + cancel() + + if err != nil { + if err.Error() == "etcdserver: role name not found" { + ctx, cancel = context.WithTimeout(context.Background(), 5*time.Second) + + _, err = auth.RoleAdd(ctx, "root") + if err != nil { + logger.Error(err, "failed to add role", "role name", "root") + return ctrl.Result{}, err + } else { + logger.Info("role added", "role name", "root") + } + cancel() + } else { + logger.Error(err, "failed to get role", "role name", "root") + return ctrl.Result{}, err + } + } else { + logger.Info("role exists, nothing to do", "role name", "root") + } + + // Create root user + ctx, cancel = context.WithTimeout(context.Background(), 5*time.Second) + rootUserResponse, err := auth.UserGet(ctx, "root") + cancel() + + if err != nil { + if err.Error() == "etcdserver: user name not found" { + ctx, cancel = context.WithTimeout(context.Background(), 5*time.Second) + + _, err = auth.UserAddWithOptions(ctx, "root", "", &clientv3.UserAddOptions{ + NoPassword: true, + }) + if err != nil { + logger.Error(err, "failed to add user", "user name", "root") + return ctrl.Result{}, err + } else { + logger.Info("user added", "user name", "root") + } + cancel() + } else { + logger.Error(err, "failed to get user", "user name", "root") + return ctrl.Result{}, err + } + } else { + logger.Info("user exists, nothing to do", "user name", "root") + } + + // Grant root role to root user + if rootUserResponse == nil || !slices.Contains(rootUserResponse.Roles, "root") { + ctx, cancel = context.WithTimeout(context.Background(), 5*time.Second) + _, err := auth.UserGrantRole(ctx, "root", "root") + cancel() + if err != nil { + logger.Error(err, "failed to grant user to role", "user:role name", "root:root") + return ctrl.Result{}, err + } + logger.Info("user:role granted", "user:role name", "root:root") + } else { + logger.Info("user:role already granted, nothing to do", "user:role name", "root:root") + } + + ctx, cancel = context.WithTimeout(context.Background(), 5*time.Second) + + // Enable auth + _, err = auth.AuthEnable(ctx) + cancel() + if err != nil { + logger.Error(err, "failed to enable auth") + return ctrl.Result{}, err + } else { + logger.Info("auth enabled") + } + } else { + // Disable auth + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + _, err = auth.AuthDisable(ctx) + cancel() + if err != nil { + logger.Error(err, "failed to disable auth") + return ctrl.Result{}, err + } else { + logger.Info("auth disabled") + } + } + + reconcileResult, err = testMemberList(ctx, cli) + if err != nil { + return reconcileResult, err + } + + return ctrl.Result{}, nil +} + +// This is auxiliary self-test function, that shows that connection to etcd cluster works. +// As soon as operator has functionality to operate etcd-cluster, this function can be removed. +func testMemberList(ctx context.Context, cli *clientv3.Client) (ctrl.Result, error) { + logger := log.FromContext(ctx) + + etcdCluster := clientv3.NewCluster(cli) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + memberList, err := etcdCluster.MemberList(ctx) + cancel() + + if err != nil { + logger.Error(err, "failed to get member list", "endpoints", cli.Endpoints()) + return ctrl.Result{}, err + } else { + logger.Info("member list got", "member list", memberList) + } + return ctrl.Result{}, nil +} + +func (r *EtcdClusterReconciler) getEtcdClient(ctx context.Context, cluster *etcdaenixiov1alpha1.EtcdCluster) (*clientv3.Client, ctrl.Result, error) { + logger := log.FromContext(ctx) + var err error + + operatorNamespace := os.Getenv("POD_NAMESPACE") + + endpoints := getEndpointsSlice(cluster) + logger.Info("endpoints built", "endpoints", endpoints) + + caCertPool := &x509.CertPool{} + cert := tls.Certificate{} + + if etcdaenixiov1alpha1.IsServerCADefined(cluster) { + + serverCASecret := &corev1.Secret{} + + if err = r.Get(ctx, client.ObjectKey{Namespace: operatorNamespace, Name: cluster.Spec.Security.TLS.ServerTrustedCASecret}, serverCASecret); err != nil { + logger.Error(err, "failed to get server trusted CA secret") + return nil, ctrl.Result{}, err + } else { + logger.Info("secret read", "server trusted CA secret", serverCASecret) + } + + pemByte := serverCASecret.Data["tls.crt"] + + caCertPool = x509.NewCertPool() + + for { + var block *pem.Block + block, pemByte = pem.Decode(pemByte) + if block == nil { + break + } + caCert, err := x509.ParseCertificate(block.Bytes) + if err != nil { + logger.Error(err, "failed to parse CA certificate") + return nil, ctrl.Result{}, err + } + + caCertPool.AddCert(caCert) + } + } + + if etcdaenixiov1alpha1.IsClientSecurityEnabled(cluster) { + + rootSecret := &corev1.Secret{} + if err = r.Get(ctx, client.ObjectKey{Namespace: operatorNamespace, Name: cluster.Spec.Security.TLS.ClientSecret}, rootSecret); err != nil { + logger.Error(err, "failed to get root client secret") + return nil, ctrl.Result{}, err + } else { + logger.Info("secret read", "root client secret", rootSecret) + } + + cert, err = tls.X509KeyPair(rootSecret.Data["tls.crt"], rootSecret.Data["tls.key"]) + if err != nil { + logger.Error(err, "failed to parse key pair", "cert", cert) + return nil, ctrl.Result{}, err + } + } + + cli, err := clientv3.New(clientv3.Config{ + Endpoints: endpoints, + DialTimeout: 5 * time.Second, + TLS: &tls.Config{ + InsecureSkipVerify: !etcdaenixiov1alpha1.IsServerCADefined(cluster), + RootCAs: caCertPool, + Certificates: []tls.Certificate{ + cert, + }, + }, + }) + if err != nil { + logger.Error(err, "failed to create etcd client", "endpoints", endpoints) + return nil, ctrl.Result{}, err + } else { + logger.Info("etcd client created", "endpoints", endpoints) + } + + return cli, ctrl.Result{}, nil + +} + +func getEndpointsSlice(cluster *etcdaenixiov1alpha1.EtcdCluster) []string { + + endpoints := []string{} + for podNumber := 1; podNumber < 3; podNumber++ { + endpoints = append( + endpoints, + strings.Join( + []string{ + factory.GetServerProtocol(cluster) + cluster.Name + "-" + strconv.Itoa(podNumber), + factory.GetHeadlessServiceName(cluster), + cluster.Namespace, + "svc:2379"}, + ".")) + } + return endpoints +} diff --git a/internal/controller/factory/statefulset.go b/internal/controller/factory/statefulset.go index 9266ebbb..33396f21 100644 --- a/internal/controller/factory/statefulset.go +++ b/internal/controller/factory/statefulset.go @@ -293,19 +293,17 @@ func generateEtcdArgs(cluster *etcdaenixiov1alpha1.EtcdCluster) []string { } serverTlsSettings := []string{} - serverProtocol := "http" if cluster.Spec.Security != nil && cluster.Spec.Security.TLS.ServerSecret != "" { serverTlsSettings = []string{ "--cert-file=/etc/etcd/pki/server/cert/tls.crt", "--key-file=/etc/etcd/pki/server/cert/tls.key", } - serverProtocol = "https" } clientTlsSettings := []string{} - if cluster.Spec.Security != nil && cluster.Spec.Security.TLS.ClientSecret != "" { + if etcdaenixiov1alpha1.IsClientSecurityEnabled(cluster) { clientTlsSettings = []string{ "--trusted-ca-file=/etc/etcd/pki/client/ca/ca.crt", "--client-cert-auth", @@ -321,10 +319,10 @@ func generateEtcdArgs(cluster *etcdaenixiov1alpha1.EtcdCluster) []string { "--name=$(POD_NAME)", "--listen-metrics-urls=http://0.0.0.0:2381", "--listen-peer-urls=https://0.0.0.0:2380", - fmt.Sprintf("--listen-client-urls=%s://0.0.0.0:2379", serverProtocol), + fmt.Sprintf("--listen-client-urls=%s0.0.0.0:2379", GetServerProtocol(cluster)), fmt.Sprintf("--initial-advertise-peer-urls=https://$(POD_NAME).%s.$(POD_NAMESPACE).svc:2380", GetHeadlessServiceName(cluster)), "--data-dir=/var/run/etcd/default.etcd", - fmt.Sprintf("--advertise-client-urls=%s://$(POD_NAME).%s.$(POD_NAMESPACE).svc:2379", serverProtocol, GetHeadlessServiceName(cluster)), + fmt.Sprintf("--advertise-client-urls=%s$(POD_NAME).%s.$(POD_NAMESPACE).svc:2379", GetServerProtocol(cluster), GetHeadlessServiceName(cluster)), }...) args = append(args, peerTlsSettings...) @@ -418,3 +416,11 @@ func getLivenessProbe() *corev1.Probe { PeriodSeconds: 5, } } + +func GetServerProtocol(cluster *etcdaenixiov1alpha1.EtcdCluster) string { + serverProtocol := "http://" + if etcdaenixiov1alpha1.IsServerSecurityEnabled(cluster) { + serverProtocol = "https://" + } + return serverProtocol +}