From 5f9a8f542cefa0a208a1b75c93609be47c663f15 Mon Sep 17 00:00:00 2001 From: Mircea Cosbuc Date: Wed, 2 Mar 2022 11:55:02 +0000 Subject: [PATCH 1/6] Watch for changes in CA certificate --- controllers/mongodb_tls.go | 7 +++ controllers/replica_set_controller.go | 20 ++++---- .../replica_set_tls_rotate_test.go | 11 +++++ test/e2e/tlstests/tlstests.go | 48 +++++++++++-------- 4 files changed, 59 insertions(+), 27 deletions(-) diff --git a/controllers/mongodb_tls.go b/controllers/mongodb_tls.go index 80c100851..0cb5c4c0f 100644 --- a/controllers/mongodb_tls.go +++ b/controllers/mongodb_tls.go @@ -91,6 +91,13 @@ func (r *ReplicaSetReconciler) validateTLSConfig(mdb mdbv1.MongoDBCommunity) (bo // Watch certificate-key secret to handle rotations r.secretWatcher.Watch(mdb.TLSSecretNamespacedName(), mdb.NamespacedName()) + // Watch CA certificate changes + if caResourceType == "Secret" { + r.secretWatcher.Watch(mdb.TLSCaCertificateSecretNamespacedName(), mdb.NamespacedName()) + } else { + r.configMapWatcher.Watch(mdb.TLSConfigMapNamespacedName(), mdb.NamespacedName()) + } + r.log.Infof("Successfully validated TLS config") return true, nil } diff --git a/controllers/replica_set_controller.go b/controllers/replica_set_controller.go index feeaef451..157c5bed0 100644 --- a/controllers/replica_set_controller.go +++ b/controllers/replica_set_controller.go @@ -72,12 +72,14 @@ func init() { func NewReconciler(mgr manager.Manager) *ReplicaSetReconciler { mgrClient := mgr.GetClient() secretWatcher := watch.New() + configMapWatcher := watch.New() return &ReplicaSetReconciler{ - client: kubernetesClient.NewClient(mgrClient), - scheme: mgr.GetScheme(), - log: zap.S(), - secretWatcher: &secretWatcher, + client: kubernetesClient.NewClient(mgrClient), + scheme: mgr.GetScheme(), + log: zap.S(), + secretWatcher: &secretWatcher, + configMapWatcher: &configMapWatcher, } } @@ -87,6 +89,7 @@ func (r *ReplicaSetReconciler) SetupWithManager(mgr ctrl.Manager) error { WithOptions(controller.Options{MaxConcurrentReconciles: 3}). For(&mdbv1.MongoDBCommunity{}, builder.WithPredicates(predicates.OnlyOnSpecChange())). Watches(&source.Kind{Type: &corev1.Secret{}}, r.secretWatcher). + Watches(&source.Kind{Type: &corev1.ConfigMap{}}, r.configMapWatcher). Owns(&appsv1.StatefulSet{}). Complete(r) } @@ -95,10 +98,11 @@ func (r *ReplicaSetReconciler) SetupWithManager(mgr ctrl.Manager) error { type ReplicaSetReconciler struct { // This client, initialized using mgr.Client() above, is a split client // that reads objects from the cache and writes to the apiserver - client kubernetesClient.Client - scheme *runtime.Scheme - log *zap.SugaredLogger - secretWatcher *watch.ResourceWatcher + client kubernetesClient.Client + scheme *runtime.Scheme + log *zap.SugaredLogger + secretWatcher *watch.ResourceWatcher + configMapWatcher *watch.ResourceWatcher } // +kubebuilder:rbac:groups=mongodbcommunity.mongodb.com,resources=mongodbcommunity,verbs=get;list;watch;create;update;patch;delete diff --git a/test/e2e/replica_set_tls_rotate/replica_set_tls_rotate_test.go b/test/e2e/replica_set_tls_rotate/replica_set_tls_rotate_test.go index 554619008..704295a76 100644 --- a/test/e2e/replica_set_tls_rotate/replica_set_tls_rotate_test.go +++ b/test/e2e/replica_set_tls_rotate/replica_set_tls_rotate_test.go @@ -58,5 +58,16 @@ func TestReplicaSetTLSRotate(t *testing.T) { defer tester.StartBackgroundConnectivityTest(t, time.Second*10, WithTls(mdb))() t.Run("Update certificate secret", tlstests.RotateCertificate(&mdb)) t.Run("Wait for certificate to be rotated", tester.WaitForRotatedCertificate(mdb, initialCertSerialNumber)) + t.Run("Wait for MongoDB to reach Running Phase after rotating server cert", mongodbtests.MongoDBReachesRunningPhase(&mdb)) + t.Run("Update CA certificate secret", tlstests.RotateCACertificate(&mdb)) + t.Run("Wait for MongoDB to reach Running Phase after rotating CA", mongodbtests.MongoDBReachesRunningPhase(&mdb)) + clientCert, err := GetClientCert(mdb) + if err != nil { + t.Fatal(err) + } + certSerialNumber := clientCert.SerialNumber + t.Run("Update certificate secret", tlstests.RotateCertificate(&mdb)) + t.Run("Wait for certificate to be rotated again", tester.WaitForRotatedCertificate(mdb, certSerialNumber)) + t.Run("Wait for MongoDB to reach Running Phase after rotating CA and server cert", mongodbtests.MongoDBReachesRunningPhase(&mdb)) }) } diff --git a/test/e2e/tlstests/tlstests.go b/test/e2e/tlstests/tlstests.go index e2c1f02ef..7966863f4 100644 --- a/test/e2e/tlstests/tlstests.go +++ b/test/e2e/tlstests/tlstests.go @@ -28,24 +28,34 @@ func EnableTLS(mdb *mdbv1.MongoDBCommunity, optional bool) func(*testing.T) { func RotateCertificate(mdb *mdbv1.MongoDBCommunity) func(*testing.T) { return func(t *testing.T) { - certKeySecretName := types.NamespacedName{Name: mdb.Spec.Security.TLS.CertificateKeySecret.Name, Namespace: mdb.Namespace} - - currentSecret := corev1.Secret{} - err := e2eutil.TestClient.Get(context.TODO(), certKeySecretName, ¤tSecret) - assert.NoError(t, err) - - // delete current cert secret, cert-manager should generate a new one - err = e2eutil.TestClient.Delete(context.TODO(), ¤tSecret) - assert.NoError(t, err) - - newSecret := corev1.Secret{} - err = wait.Poll(5*time.Second, 1*time.Minute, func() (done bool, err error) { - if err := e2eutil.TestClient.Get(context.TODO(), certKeySecretName, &newSecret); err != nil { - return false, nil - } - return true, nil - }) - assert.NoError(t, err) - assert.False(t, bytes.Equal(currentSecret.Data["tls.crt"], newSecret.Data["tls.crt"])) + certKeySecretName := mdb.TLSSecretNamespacedName() + rotateCertManagerSecret(certKeySecretName, t) + } +} + +func RotateCACertificate(mdb *mdbv1.MongoDBCommunity) func(*testing.T) { + return func(t *testing.T) { + caCertSecretName := mdb.TLSCaCertificateSecretNamespacedName() + rotateCertManagerSecret(caCertSecretName, t) } } + +func rotateCertManagerSecret(secretName types.NamespacedName, t *testing.T) { + currentSecret := corev1.Secret{} + err := e2eutil.TestClient.Get(context.TODO(), secretName, ¤tSecret) + assert.NoError(t, err) + + // delete current cert secret, cert-manager should generate a new one + err = e2eutil.TestClient.Delete(context.TODO(), ¤tSecret) + assert.NoError(t, err) + + newSecret := corev1.Secret{} + err = wait.Poll(5*time.Second, 1*time.Minute, func() (done bool, err error) { + if err := e2eutil.TestClient.Get(context.TODO(), secretName, &newSecret); err != nil { + return false, nil + } + return true, nil + }) + assert.NoError(t, err) + assert.False(t, bytes.Equal(currentSecret.Data[corev1.TLSCertKey], newSecret.Data[corev1.TLSCertKey])) +} From 2f86470d7c03cfc023ea840785e5a5379eeec602 Mon Sep 17 00:00:00 2001 From: Mircea Cosbuc Date: Tue, 8 Mar 2022 09:31:03 +0100 Subject: [PATCH 2/6] Watch for CA secret changes --- api/v1/mongodbcommunity_types.go | 4 + controllers/mongodb_tls.go | 93 ++++++++++++------- controllers/mongodb_tls_test.go | 19 ++-- controllers/replica_set_controller.go | 6 +- deploy/e2e/role.yaml | 1 + test/e2e/client.go | 12 ++- .../replica_set_tls_rotate_test.go | 12 +-- test/e2e/tlstests/tlstests.go | 30 ++++++ 8 files changed, 120 insertions(+), 57 deletions(-) diff --git a/api/v1/mongodbcommunity_types.go b/api/v1/mongodbcommunity_types.go index 57e9bc463..93f06fc78 100644 --- a/api/v1/mongodbcommunity_types.go +++ b/api/v1/mongodbcommunity_types.go @@ -726,6 +726,10 @@ func (m MongoDBCommunity) PrometheusTLSSecretNamespacedName() types.NamespacedNa return types.NamespacedName{Name: m.Spec.Prometheus.TLSSecretRef.Name, Namespace: m.Namespace} } +func (m MongoDBCommunity) TLSOperatorCASecretNamespacedName() types.NamespacedName { + return types.NamespacedName{Name: m.Name + "-ca-certificate", Namespace: m.Namespace} +} + // TLSOperatorSecretNamespacedName will get the namespaced name of the Secret created by the operator // containing the combined certificate and key. func (m MongoDBCommunity) TLSOperatorSecretNamespacedName() types.NamespacedName { diff --git a/controllers/mongodb_tls.go b/controllers/mongodb_tls.go index 0cb5c4c0f..83d12fb09 100644 --- a/controllers/mongodb_tls.go +++ b/controllers/mongodb_tls.go @@ -7,6 +7,7 @@ import ( "github.com/mongodb/mongodb-kubernetes-operator/controllers/construct" "github.com/mongodb/mongodb-kubernetes-operator/pkg/automationconfig" + "github.com/pkg/errors" "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/secret" @@ -15,7 +16,6 @@ import ( "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/podtemplatespec" "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/statefulset" - v1 "k8s.io/api/core/v1" apiErrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" @@ -40,35 +40,17 @@ func (r *ReplicaSetReconciler) validateTLSConfig(mdb mdbv1.MongoDBCommunity) (bo r.log.Info("Ensuring TLS is correctly configured") // Ensure CA cert is configured - var caResourceName types.NamespacedName - var caResourceType string - var caData map[string]string - var err error - if mdb.Spec.Security.TLS.CaCertificateSecret != nil { - caResourceName = mdb.TLSCaCertificateSecretNamespacedName() - caResourceType = "Secret" - caData, err = secret.ReadStringData(r.client, caResourceName) - } else { - caResourceName = mdb.TLSConfigMapNamespacedName() - caResourceType = "ConfigMap" - caData, err = configmap.ReadData(r.client, caResourceName) - } + _, err := getCaCrt(r.client, r.client, mdb) if err != nil { if apiErrors.IsNotFound(err) { - r.log.Warnf(`CA %s "%s" not found`, caResourceType, caResourceName) + r.log.Warnf("CA resource not found: %s", err) return false, nil } return false, err } - // Ensure Secret or ConfigMap has a "ca.crt" field - if cert, ok := caData[tlsCACertName]; !ok || cert == "" { - r.log.Warnf(`%s "%s" should have a CA certificate in field "%s"`, caResourceType, caResourceName, tlsCACertName) - return false, nil - } - // Ensure Secret exists _, err = secret.ReadStringData(r.client, mdb.TLSSecretNamespacedName()) if err != nil { @@ -92,7 +74,7 @@ func (r *ReplicaSetReconciler) validateTLSConfig(mdb mdbv1.MongoDBCommunity) (bo r.secretWatcher.Watch(mdb.TLSSecretNamespacedName(), mdb.NamespacedName()) // Watch CA certificate changes - if caResourceType == "Secret" { + if mdb.Spec.Security.TLS.CaCertificateSecret != nil { r.secretWatcher.Watch(mdb.TLSCaCertificateSecretNamespacedName(), mdb.NamespacedName()) } else { r.configMapWatcher.Watch(mdb.TLSConfigMapNamespacedName(), mdb.NamespacedName()) @@ -104,17 +86,22 @@ func (r *ReplicaSetReconciler) validateTLSConfig(mdb mdbv1.MongoDBCommunity) (bo // getTLSConfigModification creates a modification function which enables TLS in the automation config. // It will also ensure that the combined cert-key secret is created. -func getTLSConfigModification(getUpdateCreator secret.GetUpdateCreator, mdb mdbv1.MongoDBCommunity) (automationconfig.Modification, error) { +func getTLSConfigModification(cmGetter configmap.Getter, secretGetter secret.Getter, mdb mdbv1.MongoDBCommunity) (automationconfig.Modification, error) { if !mdb.Spec.Security.TLS.Enabled { return automationconfig.NOOP(), nil } - certKey, err := getPemOrConcatenatedCrtAndKey(getUpdateCreator, mdb, mdb.TLSSecretNamespacedName()) + caCert, err := getCaCrt(cmGetter, secretGetter, mdb) + if err != nil { + return automationconfig.NOOP(), err + } + + certKey, err := getPemOrConcatenatedCrtAndKey(secretGetter, mdb, mdb.TLSSecretNamespacedName()) if err != nil { return automationconfig.NOOP(), err } - return tlsConfigModification(mdb, certKey), nil + return tlsConfigModification(mdb, certKey, caCert), nil } // getCertAndKey will fetch the certificate and key from the user-provided Secret. @@ -169,6 +156,51 @@ func getPemOrConcatenatedCrtAndKey(getter secret.Getter, mdb mdbv1.MongoDBCommun return certKey, nil } +func getCaCrt(cmGetter configmap.Getter, secretGetter secret.Getter, mdb mdbv1.MongoDBCommunity) (string, error) { + var caResourceName types.NamespacedName + var caResourceType string + var caData map[string]string + var err error + if mdb.Spec.Security.TLS.CaCertificateSecret != nil { + caResourceName = mdb.TLSCaCertificateSecretNamespacedName() + caResourceType = "Secret" + caData, err = secret.ReadStringData(secretGetter, caResourceName) + } else { + caResourceName = mdb.TLSConfigMapNamespacedName() + caResourceType = "ConfigMap" + caData, err = configmap.ReadData(cmGetter, caResourceName) + } + if err != nil { + return "", err + } + + if cert, ok := caData[tlsCACertName]; !ok || cert == "" { + return "", errors.Errorf(`%s "%s" should have a CA certificate in field "%s"`, caResourceType, caResourceName, tlsCACertName) + } else { + return cert, nil + } +} + +// ensureCASecret will create or update the operator managed Secret containing +// the CA certficate from the user provided Secret or ConfigMap. +func ensureCASecret(cmGetter configmap.Getter, secretGetter secret.Getter, getUpdateCreator secret.GetUpdateCreator, mdb mdbv1.MongoDBCommunity) error { + cert, err := getCaCrt(cmGetter, secretGetter, mdb) + if err != nil { + return err + } + + caFileName := tlsOperatorSecretFileName(cert) + + operatorSecret := secret.Builder(). + SetName(mdb.TLSOperatorCASecretNamespacedName().Name). + SetNamespace(mdb.TLSOperatorCASecretNamespacedName().Namespace). + SetField(caFileName, cert). + SetOwnerReferences(mdb.GetOwnerReferences()). + Build() + + return secret.CreateOrUpdate(getUpdateCreator, operatorSecret) +} + // ensureTLSSecret will create or update the operator-managed Secret containing // the concatenated certificate and key from the user-provided Secret. func ensureTLSSecret(getUpdateCreator secret.GetUpdateCreator, mdb mdbv1.MongoDBCommunity) error { @@ -221,8 +253,8 @@ func tlsOperatorSecretFileName(certKey string) string { } // tlsConfigModification will enable TLS in the automation config. -func tlsConfigModification(mdb mdbv1.MongoDBCommunity, certKey string) automationconfig.Modification { - caCertificatePath := tlsCAMountPath + tlsCACertName +func tlsConfigModification(mdb mdbv1.MongoDBCommunity, certKey, caCert string) automationconfig.Modification { + caCertificatePath := tlsCAMountPath + tlsOperatorSecretFileName(caCert) certificateKeyPath := tlsOperatorSecretMountPath + tlsOperatorSecretFileName(certKey) mode := automationconfig.TLSModeRequired @@ -254,12 +286,7 @@ func buildTLSPodSpecModification(mdb mdbv1.MongoDBCommunity) podtemplatespec.Mod // Configure a volume which mounts the CA certificate from either a Secret or a ConfigMap // The certificate is used by both mongod and the agent - var caVolume v1.Volume - if mdb.Spec.Security.TLS.CaCertificateSecret != nil { - caVolume = statefulset.CreateVolumeFromSecret("tls-ca", mdb.Spec.Security.TLS.CaCertificateSecret.Name) - } else { - caVolume = statefulset.CreateVolumeFromConfigMap("tls-ca", mdb.Spec.Security.TLS.CaConfigMap.Name) - } + caVolume := statefulset.CreateVolumeFromSecret("tls-ca", mdb.TLSOperatorCASecretNamespacedName().Name) caVolumeMount := statefulset.CreateVolumeMount(caVolume.Name, tlsCAMountPath, statefulset.WithReadOnly(true)) // Configure a volume which mounts the secret holding the server key and certificate diff --git a/controllers/mongodb_tls_test.go b/controllers/mongodb_tls_test.go index 82a503587..cde48914f 100644 --- a/controllers/mongodb_tls_test.go +++ b/controllers/mongodb_tls_test.go @@ -39,17 +39,16 @@ func TestStatefulSet_IsCorrectlyConfiguredWithTLS(t *testing.T) { // Assert that all TLS volumes have been added. assert.Len(t, sts.Spec.Template.Spec.Volumes, 7) + permission := int32(416) assert.Contains(t, sts.Spec.Template.Spec.Volumes, corev1.Volume{ Name: "tls-ca", VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: mdb.Spec.Security.TLS.CaConfigMap.Name, - }, + Secret: &corev1.SecretVolumeSource{ + SecretName: mdb.TLSOperatorCASecretNamespacedName().Name, + DefaultMode: &permission, }, }, }) - permission := int32(416) assert.Contains(t, sts.Spec.Template.Spec.Volumes, corev1.Volume{ Name: "tls-secret", VolumeSource: corev1.VolumeSource{ @@ -90,7 +89,7 @@ func TestAutomationConfig_IsCorrectlyConfiguredWithTLS(t *testing.T) { err = createTLSConfigMap(client, mdb) assert.NoError(t, err) - tlsModification, err := getTLSConfigModification(client, mdb) + tlsModification, err := getTLSConfigModification(client, client, mdb) assert.NoError(t, err) ac, err := buildAutomationConfig(mdb, automationconfig.Auth{}, automationconfig.AutomationConfig{}, tlsModification) assert.NoError(t, err) @@ -117,7 +116,7 @@ func TestAutomationConfig_IsCorrectlyConfiguredWithTLS(t *testing.T) { ac := createAC(mdb) assert.Equal(t, &automationconfig.TLS{ - CAFilePath: tlsCAMountPath + tlsCACertName, + CAFilePath: tlsCAMountPath + tlsOperatorSecretFileName("CERT"), ClientCertificateMode: automationconfig.ClientCertificateModeOptional, }, ac.TLSConfig) @@ -126,7 +125,7 @@ func TestAutomationConfig_IsCorrectlyConfiguredWithTLS(t *testing.T) { assert.Equal(t, automationconfig.TLSModeRequired, process.Args26.Get("net.tls.mode").Data()) assert.Equal(t, tlsOperatorSecretMountPath+operatorSecretFileName, process.Args26.Get("net.tls.certificateKeyFile").Data()) - assert.Equal(t, tlsCAMountPath+tlsCACertName, process.Args26.Get("net.tls.CAFile").Data()) + assert.Equal(t, tlsCAMountPath+tlsOperatorSecretFileName("CERT"), process.Args26.Get("net.tls.CAFile").Data()) assert.True(t, process.Args26.Get("net.tls.allowConnectionsWithoutCertificates").MustBool()) } }) @@ -137,7 +136,7 @@ func TestAutomationConfig_IsCorrectlyConfiguredWithTLS(t *testing.T) { ac := createAC(mdb) assert.Equal(t, &automationconfig.TLS{ - CAFilePath: tlsCAMountPath + tlsCACertName, + CAFilePath: tlsCAMountPath + tlsOperatorSecretFileName("CERT"), ClientCertificateMode: automationconfig.ClientCertificateModeOptional, }, ac.TLSConfig) @@ -146,7 +145,7 @@ func TestAutomationConfig_IsCorrectlyConfiguredWithTLS(t *testing.T) { assert.Equal(t, automationconfig.TLSModePreferred, process.Args26.Get("net.tls.mode").Data()) assert.Equal(t, tlsOperatorSecretMountPath+operatorSecretFileName, process.Args26.Get("net.tls.certificateKeyFile").Data()) - assert.Equal(t, tlsCAMountPath+tlsCACertName, process.Args26.Get("net.tls.CAFile").Data()) + assert.Equal(t, tlsCAMountPath+tlsOperatorSecretFileName("CERT"), process.Args26.Get("net.tls.CAFile").Data()) assert.True(t, process.Args26.Get("net.tls.allowConnectionsWithoutCertificates").MustBool()) } }) diff --git a/controllers/replica_set_controller.go b/controllers/replica_set_controller.go index 157c5bed0..0df61672a 100644 --- a/controllers/replica_set_controller.go +++ b/controllers/replica_set_controller.go @@ -301,6 +301,10 @@ func (r *ReplicaSetReconciler) ensureTLSResources(mdb mdbv1.MongoDBCommunity) er // the TLS secret needs to be created beforehand, as both the StatefulSet and AutomationConfig // require the contents. if mdb.Spec.Security.TLS.Enabled { + r.log.Infof("TLS is enabled, creating/updating CA secret") + if err := ensureCASecret(r.client, r.client, r.client, mdb); err != nil { + return errors.Errorf("could not ensure CA secret: %s", err) + } r.log.Infof("TLS is enabled, creating/updating TLS secret") if err := ensureTLSSecret(r.client, mdb); err != nil { return errors.Errorf("could not ensure TLS secret: %s", err) @@ -601,7 +605,7 @@ func getCustomRolesModification(mdb mdbv1.MongoDBCommunity) (automationconfig.Mo } func (r ReplicaSetReconciler) buildAutomationConfig(mdb mdbv1.MongoDBCommunity) (automationconfig.AutomationConfig, error) { - tlsModification, err := getTLSConfigModification(r.client, mdb) + tlsModification, err := getTLSConfigModification(r.client, r.client, mdb) if err != nil { return automationconfig.AutomationConfig{}, errors.Errorf("could not configure TLS modification: %s", err) } diff --git a/deploy/e2e/role.yaml b/deploy/e2e/role.yaml index f64505487..b11b12dd1 100644 --- a/deploy/e2e/role.yaml +++ b/deploy/e2e/role.yaml @@ -173,6 +173,7 @@ rules: - watch - create - update + - patch - delete - patch - deletecollection diff --git a/test/e2e/client.go b/test/e2e/client.go index 5f0ae90dd..31aaf0132 100644 --- a/test/e2e/client.go +++ b/test/e2e/client.go @@ -22,6 +22,7 @@ import ( mdbv1 "github.com/mongodb/mongodb-kubernetes-operator/api/v1" // Needed for running tests on GCP + "k8s.io/client-go/dynamic" _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" ) @@ -88,8 +89,9 @@ type E2ETestClient struct { Client client.Client // We need the core API client for some operations that the controller-runtime client doesn't support // (e.g. exec into the container) - CoreV1Client corev1client.CoreV1Client - restConfig *rest.Config + CoreV1Client corev1client.CoreV1Client + DynamicClient dynamic.Interface + restConfig *rest.Config } // NewE2ETestClient creates a new E2ETestClient. @@ -102,7 +104,11 @@ func newE2ETestClient(config *rest.Config, scheme *runtime.Scheme) (*E2ETestClie if err != nil { return nil, err } - return &E2ETestClient{Client: cli, CoreV1Client: *coreClient, restConfig: config}, err + dynamicClient, err := dynamic.NewForConfig(config) + if err != nil { + return nil, err + } + return &E2ETestClient{Client: cli, CoreV1Client: *coreClient, DynamicClient: dynamicClient, restConfig: config}, err } // Create wraps client.Create to provide post-test cleanup functionality. diff --git a/test/e2e/replica_set_tls_rotate/replica_set_tls_rotate_test.go b/test/e2e/replica_set_tls_rotate/replica_set_tls_rotate_test.go index 704295a76..ba087cfce 100644 --- a/test/e2e/replica_set_tls_rotate/replica_set_tls_rotate_test.go +++ b/test/e2e/replica_set_tls_rotate/replica_set_tls_rotate_test.go @@ -59,15 +59,7 @@ func TestReplicaSetTLSRotate(t *testing.T) { t.Run("Update certificate secret", tlstests.RotateCertificate(&mdb)) t.Run("Wait for certificate to be rotated", tester.WaitForRotatedCertificate(mdb, initialCertSerialNumber)) t.Run("Wait for MongoDB to reach Running Phase after rotating server cert", mongodbtests.MongoDBReachesRunningPhase(&mdb)) - t.Run("Update CA certificate secret", tlstests.RotateCACertificate(&mdb)) - t.Run("Wait for MongoDB to reach Running Phase after rotating CA", mongodbtests.MongoDBReachesRunningPhase(&mdb)) - clientCert, err := GetClientCert(mdb) - if err != nil { - t.Fatal(err) - } - certSerialNumber := clientCert.SerialNumber - t.Run("Update certificate secret", tlstests.RotateCertificate(&mdb)) - t.Run("Wait for certificate to be rotated again", tester.WaitForRotatedCertificate(mdb, certSerialNumber)) - t.Run("Wait for MongoDB to reach Running Phase after rotating CA and server cert", mongodbtests.MongoDBReachesRunningPhase(&mdb)) + t.Run("Extend CA certificate validity", tlstests.ExtendCACertificate(&mdb)) + t.Run("Wait for MongoDB to reach Running Phase after extending CA", mongodbtests.MongoDBReachesRunningPhase(&mdb)) }) } diff --git a/test/e2e/tlstests/tlstests.go b/test/e2e/tlstests/tlstests.go index 7966863f4..a3665d420 100644 --- a/test/e2e/tlstests/tlstests.go +++ b/test/e2e/tlstests/tlstests.go @@ -3,6 +3,7 @@ package tlstests import ( "bytes" "context" + "encoding/json" "testing" "time" @@ -10,6 +11,8 @@ import ( e2eutil "github.com/mongodb/mongodb-kubernetes-operator/test/e2e" "github.com/stretchr/testify/assert" corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" ) @@ -26,6 +29,33 @@ func EnableTLS(mdb *mdbv1.MongoDBCommunity, optional bool) func(*testing.T) { } } +func ExtendCACertificate(mdb *mdbv1.MongoDBCommunity) func(*testing.T) { + return func(t *testing.T) { + certGVR := schema.GroupVersionResource{ + Group: "cert-manager.io", + Version: "v1", + Resource: "certificates", + } + caCertificateClient := e2eutil.TestClient.DynamicClient.Resource(certGVR).Namespace(mdb.Namespace) + patch := []interface{}{ + map[string]interface{}{ + "op": "replace", + "path": "/spec/duration", + "value": "8760h0m0s", + }, + map[string]interface{}{ + "op": "replace", + "path": "/spec/renewBefore", + "value": "720h0m0s", + }, + } + payload, err := json.Marshal(patch) + assert.NoError(t, err) + _, err = caCertificateClient.Patch(context.TODO(), "tls-selfsigned-ca", types.JSONPatchType, payload, metav1.PatchOptions{}) + assert.NoError(t, err) + } +} + func RotateCertificate(mdb *mdbv1.MongoDBCommunity) func(*testing.T) { return func(t *testing.T) { certKeySecretName := mdb.TLSSecretNamespacedName() From e2e52645de73c4c7cd42496d0fbdd4f5423bff9f Mon Sep 17 00:00:00 2001 From: Mircea Cosbuc Date: Tue, 8 Mar 2022 09:31:31 +0100 Subject: [PATCH 3/6] Add missing autogenerated Prometheus methods --- api/v1/zz_generated.deepcopy.go | 22 +++++++++++++++++++ ...ommunity.mongodb.com_mongodbcommunity.yaml | 6 ----- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go index 62da7cc94..81a87d726 100644 --- a/api/v1/zz_generated.deepcopy.go +++ b/api/v1/zz_generated.deepcopy.go @@ -235,6 +235,11 @@ func (in *MongoDBCommunitySpec) DeepCopyInto(out *MongoDBCommunitySpec) { *out = new(AutomationConfigOverride) (*in).DeepCopyInto(*out) } + if in.Prometheus != nil { + in, out := &in.Prometheus, &out.Prometheus + *out = new(Prometheus) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MongoDBCommunitySpec. @@ -325,6 +330,23 @@ func (in *Privilege) DeepCopy() *Privilege { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Prometheus) DeepCopyInto(out *Prometheus) { + *out = *in + out.PasswordSecretRef = in.PasswordSecretRef + out.TLSSecretRef = in.TLSSecretRef +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Prometheus. +func (in *Prometheus) DeepCopy() *Prometheus { + if in == nil { + return nil + } + out := new(Prometheus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in ReplicaSetHorizonConfiguration) DeepCopyInto(out *ReplicaSetHorizonConfiguration) { { diff --git a/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml b/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml index e16db3156..df9e6e1c1 100644 --- a/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml +++ b/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml @@ -5,12 +5,6 @@ kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.4.1 - service.binding/type: 'mongodb' - service.binding/provider: 'community' - service.binding: 'path={.metadata.name}-{.spec.users[0].db}-{.spec.users[0].name},objectType=Secret' - service.binding/connectionString: 'path={.metadata.name}-{.spec.users[0].db}-{.spec.users[0].name},objectType=Secret,sourceKey=connectionString.standardSrv' - service.binding/username: 'path={.metadata.name}-{.spec.users[0].db}-{.spec.users[0].name},objectType=Secret,sourceKey=username' - service.binding/password: 'path={.metadata.name}-{.spec.users[0].db}-{.spec.users[0].name},objectType=Secret,sourceKey=password' creationTimestamp: null name: mongodbcommunity.mongodbcommunity.mongodb.com spec: From 999a37a30c31df1ec717151ee9b26c4afbcf9169 Mon Sep 17 00:00:00 2001 From: Mircea Cosbuc Date: Tue, 8 Mar 2022 09:32:32 +0100 Subject: [PATCH 4/6] Force re-adding service binding annotations --- .../mongodbcommunity.mongodb.com_mongodbcommunity.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml b/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml index df9e6e1c1..e16db3156 100644 --- a/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml +++ b/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml @@ -5,6 +5,12 @@ kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.4.1 + service.binding/type: 'mongodb' + service.binding/provider: 'community' + service.binding: 'path={.metadata.name}-{.spec.users[0].db}-{.spec.users[0].name},objectType=Secret' + service.binding/connectionString: 'path={.metadata.name}-{.spec.users[0].db}-{.spec.users[0].name},objectType=Secret,sourceKey=connectionString.standardSrv' + service.binding/username: 'path={.metadata.name}-{.spec.users[0].db}-{.spec.users[0].name},objectType=Secret,sourceKey=username' + service.binding/password: 'path={.metadata.name}-{.spec.users[0].db}-{.spec.users[0].name},objectType=Secret,sourceKey=password' creationTimestamp: null name: mongodbcommunity.mongodbcommunity.mongodb.com spec: From 79d992270f4758a1ba1f3e8d5be9e7a96dad317e Mon Sep 17 00:00:00 2001 From: Mircea Cosbuc Date: Tue, 8 Mar 2022 10:56:10 +0100 Subject: [PATCH 5/6] Ensure CA cert secret is extended by modifying dnsnames --- test/e2e/mongodbtests/mongodbtests.go | 11 +++++++++++ .../replica_set_tls_rotate_test.go | 1 + test/e2e/tlstests/tlstests.go | 5 +++++ 3 files changed, 17 insertions(+) diff --git a/test/e2e/mongodbtests/mongodbtests.go b/test/e2e/mongodbtests/mongodbtests.go index 94776ff80..869780c40 100644 --- a/test/e2e/mongodbtests/mongodbtests.go +++ b/test/e2e/mongodbtests/mongodbtests.go @@ -229,6 +229,17 @@ func HasExpectedPersistentVolumes(volumes []corev1.PersistentVolume) func(t *tes } } +// MongoDBReachesPendingPhase ensures the MongoDB resources gets to the Pending phase +func MongoDBReachesPendingPhase(mdb *mdbv1.MongoDBCommunity) func(t *testing.T) { + return func(t *testing.T) { + err := wait.ForMongoDBToReachPhase(t, mdb, mdbv1.Pending, time.Second*15, time.Minute*2) + if err != nil { + t.Fatal(err) + } + t.Logf("MongoDB %s/%s is Pending!", mdb.Namespace, mdb.Name) + } +} + // MongoDBReachesRunningPhase ensure the MongoDB resource reaches the Running phase func MongoDBReachesRunningPhase(mdb *mdbv1.MongoDBCommunity) func(t *testing.T) { return func(t *testing.T) { diff --git a/test/e2e/replica_set_tls_rotate/replica_set_tls_rotate_test.go b/test/e2e/replica_set_tls_rotate/replica_set_tls_rotate_test.go index ba087cfce..6eb51f50c 100644 --- a/test/e2e/replica_set_tls_rotate/replica_set_tls_rotate_test.go +++ b/test/e2e/replica_set_tls_rotate/replica_set_tls_rotate_test.go @@ -60,6 +60,7 @@ func TestReplicaSetTLSRotate(t *testing.T) { t.Run("Wait for certificate to be rotated", tester.WaitForRotatedCertificate(mdb, initialCertSerialNumber)) t.Run("Wait for MongoDB to reach Running Phase after rotating server cert", mongodbtests.MongoDBReachesRunningPhase(&mdb)) t.Run("Extend CA certificate validity", tlstests.ExtendCACertificate(&mdb)) + t.Run("Wait for MongoDB to start reconciling after extending CA", mongodbtests.MongoDBReachesPendingPhase(&mdb)) t.Run("Wait for MongoDB to reach Running Phase after extending CA", mongodbtests.MongoDBReachesRunningPhase(&mdb)) }) } diff --git a/test/e2e/tlstests/tlstests.go b/test/e2e/tlstests/tlstests.go index a3665d420..6d5632841 100644 --- a/test/e2e/tlstests/tlstests.go +++ b/test/e2e/tlstests/tlstests.go @@ -48,6 +48,11 @@ func ExtendCACertificate(mdb *mdbv1.MongoDBCommunity) func(*testing.T) { "path": "/spec/renewBefore", "value": "720h0m0s", }, + map[string]interface{}{ + "op": "add", + "path": "/spec/dnsNames", + "value": []string{"*.ca-example.domain"}, + }, } payload, err := json.Marshal(patch) assert.NoError(t, err) From f4c985e28f68c9e0f691106a02677a78db6dc39c Mon Sep 17 00:00:00 2001 From: Mircea Cosbuc Date: Tue, 8 Mar 2022 11:05:13 +0100 Subject: [PATCH 6/6] Remove usage of caResourceType --- controllers/mongodb_tls.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/controllers/mongodb_tls.go b/controllers/mongodb_tls.go index 83d12fb09..aea4e0b47 100644 --- a/controllers/mongodb_tls.go +++ b/controllers/mongodb_tls.go @@ -158,16 +158,13 @@ func getPemOrConcatenatedCrtAndKey(getter secret.Getter, mdb mdbv1.MongoDBCommun func getCaCrt(cmGetter configmap.Getter, secretGetter secret.Getter, mdb mdbv1.MongoDBCommunity) (string, error) { var caResourceName types.NamespacedName - var caResourceType string var caData map[string]string var err error if mdb.Spec.Security.TLS.CaCertificateSecret != nil { caResourceName = mdb.TLSCaCertificateSecretNamespacedName() - caResourceType = "Secret" caData, err = secret.ReadStringData(secretGetter, caResourceName) } else { caResourceName = mdb.TLSConfigMapNamespacedName() - caResourceType = "ConfigMap" caData, err = configmap.ReadData(cmGetter, caResourceName) } if err != nil { @@ -175,7 +172,7 @@ func getCaCrt(cmGetter configmap.Getter, secretGetter secret.Getter, mdb mdbv1.M } if cert, ok := caData[tlsCACertName]; !ok || cert == "" { - return "", errors.Errorf(`%s "%s" should have a CA certificate in field "%s"`, caResourceType, caResourceName, tlsCACertName) + return "", errors.Errorf(`CA certificate resource "%s" should have a CA certificate in field "%s"`, caResourceName, tlsCACertName) } else { return cert, nil }