Merge pull request #1892 from ibihim/etcd-signer

AUTH-323: cpo/ctrl/hostedcp: create etcd-signer,rootCA bundl

control-plane-operator/controllers/hostedcontrolplane/etcd/reconcile.go

@@ -8,6 +8,8 @@ import (
 
 	hyperv1 "github.com/openshift/hypershift/api/v1beta1"
 	"github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/manifests"
+	"github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/pki"
+	"github.com/openshift/hypershift/support/certs"
 	"github.com/openshift/hypershift/support/config"
 	"github.com/openshift/hypershift/support/metrics"
 	"github.com/openshift/hypershift/support/util"
@@ -98,6 +100,16 @@ func ReconcileStatefulSet(ss *appsv1.StatefulSet, p *EtcdParams) error {
 				},
 			},
 		},
+		{
+			Name: "etcd-ca",
+			VolumeSource: corev1.VolumeSource{
+				ConfigMap: &corev1.ConfigMapVolumeSource{
+					LocalObjectReference: corev1.LocalObjectReference{
+						Name: manifests.EtcdSignerCAConfigMap(ss.Namespace).Name,
+					},
+				},
+			},
+		},
 	}
 
 	p.DeploymentConfig.ApplyToStatefulSet(ss)
@@ -146,11 +158,11 @@ func buildEtcdContainer(p *EtcdParams, namespace string) func(c *corev1.Containe
 --peer-client-cert-auth=true \
 --peer-cert-file=/etc/etcd/tls/peer/peer.crt \
 --peer-key-file=/etc/etcd/tls/peer/peer.key \
---peer-trusted-ca-file=/etc/etcd/tls/peer/peer-ca.crt \
+--peer-trusted-ca-file=/etc/etcd/tls/etcd-ca/ca.crt \
 --client-cert-auth=true \
 --cert-file=/etc/etcd/tls/server/server.crt \
 --key-file=/etc/etcd/tls/server/server.key \
---trusted-ca-file=/etc/etcd/tls/server/server-ca.crt
+--trusted-ca-file=/etc/etcd/tls/etcd-ca/ca.crt
 `
 
 		var members []string
@@ -180,6 +192,10 @@ func buildEtcdContainer(p *EtcdParams, namespace string) func(c *corev1.Containe
 				Name:      "client-tls",
 				MountPath: "/etc/etcd/tls/client",
 			},
+			{
+				Name:      "etcd-ca",
+				MountPath: "/etc/etcd/tls/etcd-ca",
+			},
 		}
 		c.Env = []corev1.EnvVar{
 			{
@@ -228,7 +244,7 @@ func buildEtcdContainer(p *EtcdParams, namespace string) func(c *corev1.Containe
 			ProbeHandler: corev1.ProbeHandler{
 				Exec: &corev1.ExecAction{
 					Command: []string{"/bin/sh", "-c",
-						"/usr/bin/etcdctl --cacert /etc/etcd/tls/client/etcd-client-ca.crt --cert /etc/etcd/tls/client/etcd-client.crt --key /etc/etcd/tls/client/etcd-client.key --endpoints=localhost:2379 endpoint health"},
+						"/usr/bin/etcdctl --cacert /etc/etcd/tls/etcd-ca/ca.crt --cert /etc/etcd/tls/client/etcd-client.crt --key /etc/etcd/tls/client/etcd-client.key --endpoints=localhost:2379 endpoint health"},
 				},
 			},
 			InitialDelaySeconds: 5,
@@ -312,21 +328,21 @@ func ReconcileServiceMonitor(sm *prometheusoperatorv1.ServiceMonitor, ownerRef c
 							LocalObjectReference: corev1.LocalObjectReference{
 								Name: manifests.EtcdClientSecret(sm.Namespace).Name,
 							},
-							Key: "etcd-client.crt",
+							Key: pki.EtcdClientCrtKey,
 						},
 					},
 					KeySecret: &corev1.SecretKeySelector{
 						LocalObjectReference: corev1.LocalObjectReference{
 							Name: manifests.EtcdClientSecret(sm.Namespace).Name,
 						},
-						Key: "etcd-client.key",
+						Key: pki.EtcdClientKeyKey,
 					},
 					CA: prometheusoperatorv1.SecretOrConfigMap{
-						Secret: &corev1.SecretKeySelector{
+						ConfigMap: &corev1.ConfigMapKeySelector{
 							LocalObjectReference: corev1.LocalObjectReference{
-								Name: manifests.EtcdClientSecret(sm.Namespace).Name,
+								Name: manifests.EtcdSignerCAConfigMap(sm.Namespace).Name,
 							},
-							Key: "etcd-client-ca.crt",
+							Key: certs.CASignerCertMapKey,
 						},
 					},
 				},

control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go

@@ -1327,6 +1327,22 @@ func (r *HostedControlPlaneReconciler) reconcilePKI(ctx context.Context, hcp *hy
 		return fmt.Errorf("failed to reconcile root CA configmap: %w", err)
 	}
 
+	// Etcd signer for all the etcd-related certs
+	etcdSignerSecret := manifests.EtcdSignerSecret(hcp.Namespace)
+	if _, err := createOrUpdate(ctx, r, etcdSignerSecret, func() error {
+		return pki.ReconcileEtcdSignerSecret(etcdSignerSecret, p.OwnerRef)
+	}); err != nil {
+		return fmt.Errorf("failed to reconcile etcd signer CA secret: %w", err)
+	}
+
+	etcdSignerCM := manifests.EtcdSignerCAConfigMap(hcp.Namespace)
+	if _, err := createOrUpdate(ctx, r, etcdSignerCM, func() error {
+		// TODO remove rootCA. rootCASecret is temporarily added for upgrade scenarios. ibihim
+		return pki.ReconcileEtcdSignerConfigMap(etcdSignerCM, p.OwnerRef, etcdSignerSecret, rootCASecret)
+	}); err != nil {
+		return fmt.Errorf("failed to reconcile etcd signer CA configmap: %w", err)
+	}
+
 	// Etcd client secret
 	etcdClientSecret := manifests.EtcdClientSecret(hcp.Namespace)
 	if _, err := createOrUpdate(ctx, r, etcdClientSecret, func() error {

control-plane-operator/controllers/hostedcontrolplane/kas/config.go

@@ -169,7 +169,7 @@ func generateConfig(p KubeAPIServerConfigParams, version semver.Version) *kcpv1.
 	args.Set("enable-aggregator-routing", "true")
 	args.Set("enable-logs-handler", "false")
 	args.Set("endpoint-reconciler-type", "lease")
-	args.Set("etcd-cafile", cpath(kasVolumeEtcdClientCert().Name, pki.EtcdClientCAKey))
+	args.Set("etcd-cafile", cpath(kasVolumeEtcdCA().Name, certs.CASignerCertMapKey))
 	args.Set("etcd-certfile", cpath(kasVolumeEtcdClientCert().Name, pki.EtcdClientCrtKey))
 	args.Set("etcd-keyfile", cpath(kasVolumeEtcdClientCert().Name, pki.EtcdClientKeyKey))
 	args.Set("etcd-prefix", "kubernetes.io")

control-plane-operator/controllers/hostedcontrolplane/kas/deployment.go

@@ -47,6 +47,7 @@ var (
 			kasVolumeAggregatorCert().Name:         "/etc/kubernetes/certs/aggregator",
 			common.VolumeAggregatorCA().Name:       "/etc/kubernetes/certs/aggregator-ca",
 			common.VolumeTotalClientCA().Name:      "/etc/kubernetes/certs/client-ca",
+			kasVolumeEtcdCA().Name:                 "/etc/kubernetes/certs/etcd-ca",
 			kasVolumeEtcdClientCert().Name:         "/etc/kubernetes/certs/etcd",
 			kasVolumeServiceAccountKey().Name:      "/etc/kubernetes/secrets/svcacct-key",
 			kasVolumeOauthMetadata().Name:          "/etc/kubernetes/oauth",
@@ -179,6 +180,7 @@ func ReconcileKubeAPIServerDeployment(deployment *appsv1.Deployment,
 				util.BuildVolume(kasVolumeAggregatorCert(), buildKASVolumeAggregatorCert),
 				util.BuildVolume(common.VolumeAggregatorCA(), common.BuildVolumeAggregatorCA),
 				util.BuildVolume(kasVolumeServiceAccountKey(), buildKASVolumeServiceAccountKey),
+				util.BuildVolume(kasVolumeEtcdCA(), buildKASVolumeEtcdCA),
 				util.BuildVolume(kasVolumeEtcdClientCert(), buildKASVolumeEtcdClientCert),
 				util.BuildVolume(kasVolumeOauthMetadata(), buildKASVolumeOauthMetadata),
 				util.BuildVolume(kasVolumeAuthTokenWebhookConfig(), buildKASVolumeAuthTokenWebhookConfig),
@@ -586,6 +588,17 @@ func buildKASVolumeEtcdClientCert(v *corev1.Volume) {
 	v.Secret.SecretName = manifests.EtcdClientSecret("").Name
 }
 
+func kasVolumeEtcdCA() *corev1.Volume {
+	return &corev1.Volume{
+		Name: "etcd-ca",
+	}
+}
+
+func buildKASVolumeEtcdCA(v *corev1.Volume) {
+	v.ConfigMap = &corev1.ConfigMapVolumeSource{}
+	v.ConfigMap.Name = manifests.EtcdSignerCAConfigMap("").Name
+}
+
 func kasVolumeOauthMetadata() *corev1.Volume {
 	return &corev1.Volume{
 		Name: "oauth-metadata",

control-plane-operator/controllers/hostedcontrolplane/manifests/pki.go

@@ -27,6 +27,15 @@ func RootCAConfigMap(ns string) *corev1.ConfigMap {
 	}
 }
 
+func EtcdSignerCAConfigMap(ns string) *corev1.ConfigMap {
+	return &corev1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "etcd-ca",
+			Namespace: ns,
+		},
+	}
+}
+
 func AggregatorClientCAConfigMap(ns string) *corev1.ConfigMap {
 	return &corev1.ConfigMap{
 		ObjectMeta: metav1.ObjectMeta{
@@ -56,6 +65,10 @@ func UserCAConfigMap(ns string) *corev1.ConfigMap {
 	}
 }
 
+func EtcdSignerSecret(ns string) *corev1.Secret {
+	return secretFor(ns, "etcd-signer")
+}
+
 func EtcdClientSecret(ns string) *corev1.Secret { return secretFor(ns, "etcd-client-tls") }
 
 func EtcdServerSecret(ns string) *corev1.Secret { return secretFor(ns, "etcd-server-tls") }

control-plane-operator/controllers/hostedcontrolplane/oapi/deployment.go

@@ -287,8 +287,8 @@ func oasVolumeEtcdClientCA() *corev1.Volume {
 }
 
 func buildOASVolumeEtcdClientCA(v *corev1.Volume) {
-	v.Secret = &corev1.SecretVolumeSource{}
-	v.Secret.SecretName = manifests.RootCASecret("").Name
+	v.ConfigMap = &corev1.ConfigMapVolumeSource{}
+	v.ConfigMap.Name = manifests.EtcdSignerCAConfigMap("").Name
 }
 
 func oasVolumeServingCert() *corev1.Volume {

control-plane-operator/controllers/hostedcontrolplane/oapi/oauth_deployment.go

@@ -184,8 +184,8 @@ func oauthVolumeEtcdClientCA() *corev1.Volume {
 }
 
 func buildOAuthVolumeEtcdClientCA(v *corev1.Volume) {
-	v.Secret = &corev1.SecretVolumeSource{}
-	v.Secret.SecretName = manifests.RootCASecret("").Name
+	v.ConfigMap = &corev1.ConfigMapVolumeSource{}
+	v.ConfigMap.Name = manifests.EtcdSignerCAConfigMap("").Name
 }
 
 func oauthVolumeServingCert() *corev1.Volume {

control-plane-operator/controllers/hostedcontrolplane/pki/ca.go

@@ -62,6 +62,14 @@ func ReconcileRootCA(secret *corev1.Secret, ownerRef config.OwnerRef) error {
 	return reconcileSelfSignedCA(secret, ownerRef, "root-ca", "openshift")
 }
 
+func ReconcileEtcdSignerSecret(secret *corev1.Secret, ownerRef config.OwnerRef) error {
+	return reconcileSelfSignedCA(secret, ownerRef, "etcd-signer", "openshift")
+}
+
+func ReconcileEtcdSignerConfigMap(cm *corev1.ConfigMap, ownerRef config.OwnerRef, etcdSigner, rootCA *corev1.Secret) error {
+	return reconcileAggregateCA(cm, ownerRef, etcdSigner, rootCA)
+}
+
 func ReconcileClusterSignerCA(secret *corev1.Secret, ownerRef config.OwnerRef) error {
 	return reconcileSelfSignedCA(secret, ownerRef, "cluster-signer", "openshift")
 }

control-plane-operator/controllers/hostedcontrolplane/pki/etcd.go

@@ -44,5 +44,6 @@ func ReconcileEtcdPeerSecret(secret, ca *corev1.Secret, ownerRef config.OwnerRef
 		fmt.Sprintf("*.etcd-discovery.%s.svc", secret.Namespace),
 		fmt.Sprintf("*.etcd-discovery.%s.svc.cluster.local", secret.Namespace),
 	}
+
 	return reconcileSignedCertWithKeysAndAddresses(secret, ca, ownerRef, "etcd-discovery", []string{"kubernetes"}, X509UsageClientServerAuth, EtcdPeerCrtKey, EtcdPeerKeyKey, EtcdPeerCAKey, dnsNames, nil)
 }