Merge pull request #386 from tnozicka/rotate-csr-signer-on-recovery-4.4

[release-4.4] Bug 1820613: Refresh csr-signer in recovery flow
openshift · Apr 9, 2020 · 568856a · 568856a
2 parents 1a267c8 + d9cb19d
commit 568856a
Show file tree

Hide file tree

Showing 3 changed files with 88 additions and 11 deletions.
diff --git a/pkg/cmd/recoverycontroller/cmd.go b/pkg/cmd/recoverycontroller/cmd.go
@@ -9,7 +9,11 @@ import (
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/pkg/version"
 
+	operatorv1 "github.com/openshift/api/operator/v1"
+	"github.com/openshift/cluster-kube-controller-manager-operator/pkg/operator/certrotationcontroller"
 	"github.com/openshift/library-go/pkg/controller/controllercmd"
+	"github.com/openshift/library-go/pkg/operator/certrotation"
+	"github.com/openshift/library-go/pkg/operator/genericoperatorclient"
 	"github.com/openshift/library-go/pkg/operator/v1helpers"
 
 	"github.com/openshift/cluster-kube-controller-manager-operator/pkg/operator/operatorclient"
@@ -78,6 +82,31 @@ func (o *Options) Run(ctx context.Context) error {
 		operatorclient.TargetNamespace,
 	)
 
+	operatorClient, dynamicInformers, err := genericoperatorclient.NewStaticPodOperatorClient(o.controllerContext.KubeConfig, operatorv1.GroupVersion.WithResource("kubecontrollermanagers"))
+	if err != nil {
+		return err
+	}
+
+	certRotationScale, err := certrotation.GetCertRotationScale(kubeClient, operatorclient.GlobalUserSpecifiedConfigNamespace)
+	if err != nil {
+		return err
+	}
+
+	certRotationController, err := certrotationcontroller.NewCertRotationControllerOnlyWhenExpired(
+		v1helpers.CachedSecretGetter(kubeClient.CoreV1(), kubeInformersForNamespaces),
+		v1helpers.CachedConfigMapGetter(kubeClient.CoreV1(), kubeInformersForNamespaces),
+		operatorClient,
+		kubeInformersForNamespaces,
+		o.controllerContext.EventRecorder,
+		// this is weird, but when we turn down rotation in CI, we go fast enough that kubelets and kas are racing to observe the new signer before the signer is used.
+		// we need to establish some kind of delay or back pressure to prevent the rollout.  This ensures we don't trigger kas restart
+		// during e2e tests for now.
+		certRotationScale*8,
+	)
+	if err != nil {
+		return err
+	}
+
 	csrController, err := NewCSRController(
 		kubeClient,
 		kubeInformersForNamespaces,
@@ -88,9 +117,13 @@ func (o *Options) Run(ctx context.Context) error {
 	}
 
 	kubeInformersForNamespaces.Start(ctx.Done())
+	dynamicInformers.Start(ctx.Done())
 
 	// FIXME: These are missing a wait group to track goroutines and handle graceful termination
 	// (@deads2k wants time to think it through)
+	go func() {
+		certRotationController.Run(ctx, 1)
+	}()
 
 	go func() {
 		csrController.Run(ctx)

diff --git a/pkg/cmd/recoverycontroller/csrcontroller.go b/pkg/cmd/recoverycontroller/csrcontroller.go
@@ -125,6 +125,9 @@ func (c *CSRController) processNextItem(ctx context.Context) bool {
 }
 
 func (c *CSRController) sync(ctx context.Context) error {
+	klog.V(4).Infof("Starting CSRController sync")
+	defer klog.V(4).Infof("CSRController sync done")
+
 	// Always start 10 seconds later after a change occurred. Makes us less likely to steal work and logs from the operator.
 	timer := time.NewTimer(10 * time.Second)
 	defer timer.Stop()

diff --git a/pkg/operator/certrotationcontroller/certrotationcontroller.go b/pkg/operator/certrotationcontroller/certrotationcontroller.go
@@ -29,6 +29,45 @@ func NewCertRotationController(
 	kubeInformersForNamespaces v1helpers.KubeInformersForNamespaces,
 	eventRecorder events.Recorder,
 	day time.Duration,
+) (*CertRotationController, error) {
+	return newCertRotationController(
+		secretsGetter,
+		configMapsGetter,
+		operatorClient,
+		kubeInformersForNamespaces,
+		eventRecorder,
+		day,
+		false,
+	)
+}
+
+func NewCertRotationControllerOnlyWhenExpired(
+	secretsGetter corev1client.SecretsGetter,
+	configMapsGetter corev1client.ConfigMapsGetter,
+	operatorClient v1helpers.StaticPodOperatorClient,
+	kubeInformersForNamespaces v1helpers.KubeInformersForNamespaces,
+	eventRecorder events.Recorder,
+	day time.Duration,
+) (*CertRotationController, error) {
+	return newCertRotationController(
+		secretsGetter,
+		configMapsGetter,
+		operatorClient,
+		kubeInformersForNamespaces,
+		eventRecorder,
+		day,
+		true,
+	)
+}
+
+func newCertRotationController(
+	secretsGetter corev1client.SecretsGetter,
+	configMapsGetter corev1client.ConfigMapsGetter,
+	operatorClient v1helpers.StaticPodOperatorClient,
+	kubeInformersForNamespaces v1helpers.KubeInformersForNamespaces,
+	eventRecorder events.Recorder,
+	day time.Duration,
+	refreshOnlyWhenExpired bool,
 ) (*CertRotationController, error) {
 	ret := &CertRotationController{}
 
@@ -44,13 +83,14 @@ func NewCertRotationController(
 		certrotation.SigningRotation{
 			Namespace: operatorclient.OperatorNamespace,
 			// this is not a typo, this is the signer of the signer
-			Name:          "csr-signer-signer",
-			Validity:      60 * rotationDay,
-			Refresh:       30 * rotationDay,
-			Informer:      kubeInformersForNamespaces.InformersFor(operatorclient.OperatorNamespace).Core().V1().Secrets(),
-			Lister:        kubeInformersForNamespaces.InformersFor(operatorclient.OperatorNamespace).Core().V1().Secrets().Lister(),
-			Client:        secretsGetter,
-			EventRecorder: eventRecorder,
+			Name:                   "csr-signer-signer",
+			Validity:               60 * rotationDay,
+			Refresh:                30 * rotationDay,
+			RefreshOnlyWhenExpired: refreshOnlyWhenExpired,
+			Informer:               kubeInformersForNamespaces.InformersFor(operatorclient.OperatorNamespace).Core().V1().Secrets(),
+			Lister:                 kubeInformersForNamespaces.InformersFor(operatorclient.OperatorNamespace).Core().V1().Secrets().Lister(),
+			Client:                 secretsGetter,
+			EventRecorder:          eventRecorder,
 		},
 		certrotation.CABundleRotation{
 			Namespace:     operatorclient.OperatorNamespace,
@@ -61,10 +101,11 @@ func NewCertRotationController(
 			EventRecorder: eventRecorder,
 		},
 		certrotation.TargetRotation{
-			Namespace: operatorclient.OperatorNamespace,
-			Name:      "csr-signer",
-			Validity:  30 * rotationDay,
-			Refresh:   15 * rotationDay,
+			Namespace:              operatorclient.OperatorNamespace,
+			Name:                   "csr-signer",
+			Validity:               30 * rotationDay,
+			Refresh:                15 * rotationDay,
+			RefreshOnlyWhenExpired: refreshOnlyWhenExpired,
 			CertCreator: &certrotation.SignerRotation{
 				SignerName: "kube-csr-signer",
 			},