ETCD-565: add manual etcd signer cert rotation e2e test

test/extended/dr/OWNERS

@@ -5,8 +5,8 @@ reviewers:
   - tjungblu
 approvers:
   - deads2k
+  - soltysh
   - hasbro17
-  - sttts
   - dusk125
   - Elbehery
   - tjungblu

test/extended/etcd/OWNERS

@@ -1,12 +1,12 @@
 reviewers:
-  - csrwng
+  - dusk125
   - hasbro17
-  - hexfusion
-  - ironcladlou
-  - smarterclayton
+  - Elbehery
+  - tjungblu
 approvers:
-  - csrwng
+  - deads2k
+  - soltysh
   - hasbro17
-  - hexfusion
-  - ironcladlou
-  - smarterclayton
+  - dusk125
+  - Elbehery
+  - tjungblu

test/extended/etcd/cert_rotation.go

@@ -0,0 +1,71 @@
+package etcd
+
+import (
+	"context"
+	g "github.com/onsi/ginkgo/v2"
+	o "github.com/onsi/gomega"
+	"github.com/openshift/library-go/test/library"
+	exutil "github.com/openshift/origin/test/extended/util"
+	"github.com/pkg/errors"
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	corev1apply "k8s.io/client-go/applyconfigurations/core/v1"
+	"time"
+)
+
+var _ = g.Describe("[sig-etcd] etcd", func() {
+	defer g.GinkgoRecover()
+
+	ctx := context.TODO()
+	oc := exutil.NewCLIWithoutNamespace("etcd-certs").AsAdmin()
+
+	g.It("can manually rotate signer certificates", func() {
+		c := oc.AdminKubeClient()
+		kasSecretsClient := c.CoreV1().Secrets("openshift-kube-apiserver")
+		kasPodClient := c.CoreV1().Pods("openshift-kube-apiserver")
+		etcdSecretsClient := c.CoreV1().Secrets("openshift-etcd")
+		etcdPodClient := c.CoreV1().Pods("openshift-etcd")
+		configSecretsClient := c.CoreV1().Secrets("openshift-config")
+
+		currentKasClientCert, err := kasSecretsClient.Get(ctx, "etcd-client", v1.GetOptions{})
+		o.Expect(err).ToNot(o.HaveOccurred())
+		currentEtcdLeafCerts, err := etcdSecretsClient.Get(ctx, "etcd-all-certs", v1.GetOptions{})
+		o.Expect(err).ToNot(o.HaveOccurred())
+
+		// as of 4.16, the manual signer rotation is effectively a secret copy, similar to below OC command:
+		// $ oc get secret etcd-signer -n openshift-etcd -ojson | \
+		// jq 'del(.metadata["namespace","creationTimestamp","resourceVersion","selfLink","uid"])' | \
+		// oc apply -n openshift-config -f -
+		newSigner, err := etcdSecretsClient.Get(ctx, "etcd-signer", v1.GetOptions{})
+		o.Expect(err).ToNot(o.HaveOccurred())
+
+		applyConf, err := corev1apply.ExtractSecret(newSigner, "cluster-etcd-operator")
+		o.Expect(err).ToNot(o.HaveOccurred())
+		applyConf = applyConf.WithNamespace("openshift-config").WithData(newSigner.Data)
+		_, err = configSecretsClient.Apply(ctx, applyConf, v1.ApplyOptions{})
+		o.Expect(err).ToNot(o.HaveOccurred())
+
+		g.GinkgoT().Log("waiting for etcd/apiserver to stabilize on the same revision")
+		// await all rollouts, then assert the leaf certs all successfully changed
+		err = library.WaitForPodsToStabilizeOnTheSameRevision(g.GinkgoT(), etcdPodClient, "etcd=true",
+			5, 1*time.Minute, 30*time.Second, 30*time.Minute)
+		err = errors.Wrap(err, "timed out waiting for etcd pods to stabilize on the same revision")
+		o.Expect(err).ToNot(o.HaveOccurred())
+
+		err = library.WaitForPodsToStabilizeOnTheSameRevision(g.GinkgoT(), kasPodClient, "apiserver=true",
+			5, 1*time.Minute, 30*time.Second, 30*time.Minute)
+		err = errors.Wrap(err, "timed out waiting for APIServer pods to stabilize on the same revision")
+		o.Expect(err).ToNot(o.HaveOccurred())
+
+		rotatedKasClientCert, err := kasSecretsClient.Get(ctx, "etcd-client", v1.GetOptions{})
+		o.Expect(err).ToNot(o.HaveOccurred())
+		o.Expect(rotatedKasClientCert.Data).ToNot(o.Equal(currentKasClientCert.Data))
+
+		rotatedEtcdLeafCerts, err := etcdSecretsClient.Get(ctx, "etcd-all-certs", v1.GetOptions{})
+		o.Expect(err).ToNot(o.HaveOccurred())
+		o.Expect(rotatedEtcdLeafCerts.Data).ToNot(o.Equal(currentEtcdLeafCerts.Data))
+	})
+
+	// TODO(thomas): delete "dynamic" certs, expect them to re-gen
+	// TODO(thomas): rotate metrics signer cert, expect prometheus to still return etcd metrics
+
+})