pkg/operator/targetconfigcontroller: wait for kcm-o to generate certs…

… before rollout Signed-off-by: Sam Batschelet <sbatsche@redhat.com>
openshift · Jun 30, 2021 · 8140e84 · 8140e84
1 parent bf303e6
commit 8140e84
Show file tree

Hide file tree

Showing 5 changed files with 197 additions and 1 deletion.
diff --git a/pkg/operator/targetconfigcontroller/targetconfigcontroller.go b/pkg/operator/targetconfigcontroller/targetconfigcontroller.go
@@ -2,6 +2,8 @@ package targetconfigcontroller
 
 import (
 	"context"
+	"crypto/x509"
+	"encoding/pem"
 	"fmt"
 	"strings"
 	"time"
@@ -114,7 +116,9 @@ func createTargetConfig(c TargetConfigController, recorder events.Recorder, oper
 	if err != nil {
 		return false, err
 	}
-
+	if err := c.checkDependencies(); err != nil {
+		errors = append(errors, err)
+	}
 	_, _, err = c.manageStandardPod(contentReplacer, c.kubeClient.CoreV1(), recorder, operatorSpec)
 	if err != nil {
 		errors = append(errors, fmt.Errorf("%q: %v", "configmap/etcd-pod", err))
@@ -180,6 +184,20 @@ func (c *TargetConfigController) getSubstitutionReplacer(operatorSpec *operatorv
 	), nil
 }
 
+func (c *TargetConfigController) checkDependencies() error {
+	// Ensure kube-controller-manager-operator has generated csr-controller-ca before rolling out.
+	// Waiting here is very important in single node as we want to avoid disruption until
+	// key resources have rotated.
+	csrControllerCAConfigMap, err := c.configMapLister.ConfigMaps(operatorclient.GlobalMachineSpecifiedConfigNamespace).Get("csr-controller-ca")
+	if err != nil {
+		return err
+	}
+	if err := ensureCSRControllerCAConfigMap(csrControllerCAConfigMap); err != nil {
+		return err
+	}
+	return nil
+}
+
 func (c *TargetConfigController) manageRecoveryPod(substitutionReplacer *strings.Replacer, client coreclientv1.ConfigMapsGetter, recorder events.Recorder, operatorSpec *operatorv1.StaticPodOperatorSpec) (*corev1.ConfigMap, bool, error) {
 	podBytes := etcd_assets.MustAsset("etcd/restore-pod.yaml")
 	substitutedPodString := substitutionReplacer.Replace(string(podBytes))
@@ -246,3 +264,38 @@ func (c *TargetConfigController) namespaceEventHandler() cache.ResourceEventHand
 		},
 	}
 }
+
+// ensureCSRControllerCA validates that the openshift-config-managed configmap csr-controller-ca contains a valid
+// CA signed by kubelet-signer. This is required to ensure rollout happens in a time that minimizes possible
+// cluster disruption in non HA deployments.
+func ensureCSRControllerCAConfigMap(csrControllerCAConfigMap *corev1.ConfigMap) error {
+	caBundle := csrControllerCAConfigMap.Data["ca-bundle.crt"]
+	if len(caBundle) == 0 {
+		return fmt.Errorf("configmap openshift-config-managed/csr-controller-ca is not valid, waiting on kube-controller-manager-operator to create")
+	}
+	block, _ := pem.Decode([]byte(caBundle))
+	leafCert, err := x509.ParseCertificate(block.Bytes)
+	if err != nil {
+		return err
+	}
+	if len(leafCert.Subject.CommonName) == 0 {
+		return fmt.Errorf("configmap openshift-config-managed/csr-controller-ca is not valid, waiting on kube-controller-manager-operator to generate")
+	}
+
+	isCASubjectExpected := strings.HasPrefix(leafCert.Subject.CommonName, "kube-csr-signer")
+	if !isCASubjectExpected {
+		return fmt.Errorf("configmap openshift-config-managed/csr-controller-ca subject common name is not valid: %q", leafCert.Subject.CommonName)
+	}
+	var isCAManagerExpected bool
+	for _, managedField := range csrControllerCAConfigMap.ManagedFields {
+		if managedField.Manager == "cluster-kube-controller-manager-operator" {
+			isCAManagerExpected = true
+		}
+	}
+
+	if !isCAManagerExpected {
+		return fmt.Errorf("configmap openshift-config-managed/csr-controller-ca manager is not valid")
+	}
+
+	return nil
+}
diff --git a/pkg/operator/targetconfigcontroller/targetconfigcontroller_test.go b/pkg/operator/targetconfigcontroller/targetconfigcontroller_test.go
@@ -0,0 +1,54 @@
+package targetconfigcontroller
+
+import (
+	"io/ioutil"
+	"testing"
+
+	"github.com/openshift/library-go/pkg/operator/resource/resourceread"
+	"github.com/stretchr/testify/require"
+)
+
+const (
+	testBootstrapGeneratedCAPath = "testdata/bootstrap-configmap-csr-controller-ca.yaml"
+	testKCMGeneratedCAPath       = "testdata/configmap-csr-controller-ca.yaml"
+	testNoDataCAPath             = "testdata/nodata-configmap-csr-controller-ca.yaml"
+)
+
+func Test_ensureCSRControllerCAConfigMap(t *testing.T) {
+	tests := []struct {
+		name          string
+		configMapPath string
+		wantErr       bool
+	}{
+		{
+			name:          "test CA materials generated by installer kubelet-client-ca-bundle.crt",
+			configMapPath: testBootstrapGeneratedCAPath,
+			wantErr:       true,
+		},
+		{
+			name:          "test CA materials generated by kube-controller-manager-operator",
+			configMapPath: testKCMGeneratedCAPath,
+			wantErr:       false,
+		},
+		{
+			name:          "test missing CA materials",
+			configMapPath: testNoDataCAPath,
+			wantErr:       true,
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			cmBytes, err := ioutil.ReadFile(test.configMapPath)
+			require.NoError(t, err)
+			caConfigMap := resourceread.ReadConfigMapV1OrDie(cmBytes)
+			gotErr := ensureCSRControllerCAConfigMap(caConfigMap)
+			if gotErr != nil && !test.wantErr {
+				t.Errorf("unexpected expected error %v", gotErr)
+			}
+			if gotErr == nil && test.wantErr {
+				t.Errorf("expected error got nil")
+			}
+		})
+	}
+
+}
diff --git a/pkg/operator/targetconfigcontroller/testdata/bootstrap-configmap-csr-controller-ca.yaml b/pkg/operator/targetconfigcontroller/testdata/bootstrap-configmap-csr-controller-ca.yaml
@@ -0,0 +1,27 @@
+# this is written by the kcm-o, but we initialize here to cleanly handle the adoption case
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: csr-controller-ca
+  namespace: openshift-config-managed
+data:
+  ca-bundle.crt: |
+    -----BEGIN CERTIFICATE-----
+    MIIDHjCCAgagAwIBAgIIPFoVWHbgcKQwDQYJKoZIhvcNAQELBQAwLTESMBAGA1UE
+    CxMJb3BlbnNoaWZ0MRcwFQYDVQQDEw5rdWJlbGV0LXNpZ25lcjAeFw0yMTA2MzAw
+    MjAxMzdaFw0yMTA3MDEwMjAxMzdaMC0xEjAQBgNVBAsTCW9wZW5zaGlmdDEXMBUG
+    A1UEAxMOa3ViZWxldC1zaWduZXIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK
+    AoIBAQDNneh+K8k75eaV1AlGSKVSsMJk8bwSijxmK10cpwnXkYejSvTD8IHnYHie
+    Lx45/OCelg71D+bZJIATbPFT5UcUXymtlzXl8miFNF6VGGeycwsa+rmGiNCn7b/O
+    QL9pOUtMi+7GVod7E8jdmQnnYaFQwXDVrA0pbz6U8fqVqBNqgxXNkySCJD0q/LMR
+    m1utoceZ1GFIagz9Rq6X3ysI2NPprsMslLJvHHuu67KA4+jY9tUgWqzZApIiWo1J
+    zsh/KJryD0yB4Zxmp9gxwZMbgkifguc1EPARiF44ASlSdrkLZiMzj6Rn9D0T7j1o
+    Xk2lAQsk7oA6JzNs4L4iJT6b6iCzAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwICpDAP
+    BgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBSuj6vBj5M6MgzZM3IxI4mTsIHA4TAN
+    BgkqhkiG9w0BAQsFAAOCAQEAkQPmoUn2fTnv6EwoL61ynfKaK2wieQnPH+Kbrrrr
+    ypceZJzEzwOToi5D1kM/P59DXMMqizyEjG4uRMSR1oSsX7qbgh5ZmIo0Z0m8uAai
+    PKGtOfup3rK/5DzxiTqO9onq5Ri2P5OlzCYj7tRO4HIVRZgnbkfhn/Abtx+tA69r
+    BUP87uqOIWwSVLaWv4kWHDyg6tDmL4zb2rNLwyHYp6Ox5VUPJPZmZHRe8SNPR3bO
+    T5XAEjPryZ53W4O12h2nCxu++4WIZwUcIkMY1EK0psw9bjEGIXY3z8OdAfcKuNNn
+    UNQlm7klBGHxxdTXaj1w2l7n/4QJkmlx8WQlIJx77yMPpg==
+    -----END CERTIFICATE-----
diff --git a/pkg/operator/targetconfigcontroller/testdata/configmap-csr-controller-ca.yaml b/pkg/operator/targetconfigcontroller/testdata/configmap-csr-controller-ca.yaml
@@ -0,0 +1,55 @@
+apiVersion: v1
+data:
+  ca-bundle.crt: |
+    -----BEGIN CERTIFICATE-----
+    MIIDODCCAiCgAwIBAgIISVOVGr6vvpgwDQYJKoZIhvcNAQELBQAwLTESMBAGA1UE
+    CxMJb3BlbnNoaWZ0MRcwFQYDVQQDEw5rdWJlbGV0LXNpZ25lcjAeFw0yMTA2MzAw
+    MjEyMDNaFw0yMTA3MDEwMjAxMzdaMCYxJDAiBgNVBAMMG2t1YmUtY3NyLXNpZ25l
+    cl9AMTYyNTAxOTEyNDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKa3
+    4+tpn0qcvT+OFnKcRaMhWLknk3tOAFRaJcUivxpPZ1byo6RuAvXKz/OPxYfjHtNP
+    Y09yXBdI75SOjOTyvxxPDmQ5tzfFZt9hDCvU8yfRn4Q6NGk3fvxFXayAcdUglxmi
+    7t+edh1fLxS+rTXz/P5h/GfXVEALj/ZVwFeEYgByIkzC+EvfQ9cUE+n8rUXj+hiZ
+    Xa34Hr6zuKfTWTjQRp6beMS27ALXwEIQhLbVa08bBNb02cUIVsmA/l7hx2UOqStf
+    JaX4w/t3J8t7/U/s/BCVSxhe3Z3jDIt/CekbXQvaYPukfm4AR6llZi3uLm0j/HNq
+    HSEwUWNEGU5/yltmYCkCAwEAAaNjMGEwDgYDVR0PAQH/BAQDAgKkMA8GA1UdEwEB
+    /wQFMAMBAf8wHQYDVR0OBBYEFJw5A5wVxCRtSSJ8HcifBawvbFchMB8GA1UdIwQY
+    MBaAFK6Pq8GPkzoyDNkzcjEjiZOwgcDhMA0GCSqGSIb3DQEBCwUAA4IBAQBNZiPt
+    /MxChdxDvlTYf1TYZUbQ0B/ABaU4j97D9ZjsE27+5uAgxzXOU549RdJi0C+64pBQ
+    WQgU0J2HcNCdLHXmu0D6LhurJyeiDxpUf1YYBOtER2G9+ZSsPkKCwRMG3aEcU4PW
+    HTUCp53HBkpQO7F7UuTpyGX7c6hAcolJ5yR0Eekket8Saign25jM3AW0oZurceOZ
+    jhmKIdltFzixadn0ndyaWz3Hg7aQrEt4Ze709cFF4PSisefgqP0Vylj6rCjLREue
+    jufbRqaucZ4w4W+L9+ofWDPVFBlxzsreTyqFFnKQuXVgoGvWFRlWepyC5dxy03Fy
+    6nhlOGfYUuBN2+eJ
+    -----END CERTIFICATE-----
+    -----BEGIN CERTIFICATE-----
+    MIIDHjCCAgagAwIBAgIIPFoVWHbgcKQwDQYJKoZIhvcNAQELBQAwLTESMBAGA1UE
+    CxMJb3BlbnNoaWZ0MRcwFQYDVQQDEw5rdWJlbGV0LXNpZ25lcjAeFw0yMTA2MzAw
+    MjAxMzdaFw0yMTA3MDEwMjAxMzdaMC0xEjAQBgNVBAsTCW9wZW5zaGlmdDEXMBUG
+    A1UEAxMOa3ViZWxldC1zaWduZXIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK
+    AoIBAQDNneh+K8k75eaV1AlGSKVSsMJk8bwSijxmK10cpwnXkYejSvTD8IHnYHie
+    Lx45/OCelg71D+bZJIATbPFT5UcUXymtlzXl8miFNF6VGGeycwsa+rmGiNCn7b/O
+    QL9pOUtMi+7GVod7E8jdmQnnYaFQwXDVrA0pbz6U8fqVqBNqgxXNkySCJD0q/LMR
+    m1utoceZ1GFIagz9Rq6X3ysI2NPprsMslLJvHHuu67KA4+jY9tUgWqzZApIiWo1J
+    zsh/KJryD0yB4Zxmp9gxwZMbgkifguc1EPARiF44ASlSdrkLZiMzj6Rn9D0T7j1o
+    Xk2lAQsk7oA6JzNs4L4iJT6b6iCzAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwICpDAP
+    BgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBSuj6vBj5M6MgzZM3IxI4mTsIHA4TAN
+    BgkqhkiG9w0BAQsFAAOCAQEAkQPmoUn2fTnv6EwoL61ynfKaK2wieQnPH+Kbrrrr
+    ypceZJzEzwOToi5D1kM/P59DXMMqizyEjG4uRMSR1oSsX7qbgh5ZmIo0Z0m8uAai
+    PKGtOfup3rK/5DzxiTqO9onq5Ri2P5OlzCYj7tRO4HIVRZgnbkfhn/Abtx+tA69r
+    BUP87uqOIWwSVLaWv4kWHDyg6tDmL4zb2rNLwyHYp6Ox5VUPJPZmZHRe8SNPR3bO
+    T5XAEjPryZ53W4O12h2nCxu++4WIZwUcIkMY1EK0psw9bjEGIXY3z8OdAfcKuNNn
+    UNQlm7klBGHxxdTXaj1w2l7n/4QJkmlx8WQlIJx77yMPpg==
+    -----END CERTIFICATE-----
+kind: ConfigMap
+metadata:
+  managedFields:
+    - apiVersion: v1
+      fieldsType: FieldsV1
+      fieldsV1:
+        f:data:
+          .: {}
+          f:ca-bundle.crt: {}
+      manager: cluster-kube-controller-manager-operator
+      operation: Update
+  name: csr-controller-ca
+  namespace: openshift-config-managed
diff --git a/pkg/operator/targetconfigcontroller/testdata/nodata-configmap-csr-controller-ca.yaml b/pkg/operator/targetconfigcontroller/testdata/nodata-configmap-csr-controller-ca.yaml
@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: csr-controller-ca
+  namespace: openshift-config-managed
+data:
+  ca-bundle.crt: