From afd4c60a9aa554c33b52f62476aafc36e960405f Mon Sep 17 00:00:00 2001 From: chatton Date: Thu, 10 Jun 2021 09:49:00 +0100 Subject: [PATCH] Move Wait functions into separate package --- test/e2e/e2eutil.go | 129 ------------------------ test/e2e/mongodbtests/mongodbtests.go | 24 +++-- test/e2e/util/wait/wait.go | 138 ++++++++++++++++++++++++++ 3 files changed, 151 insertions(+), 140 deletions(-) create mode 100644 test/e2e/util/wait/wait.go diff --git a/test/e2e/e2eutil.go b/test/e2e/e2eutil.go index 88be05e76..dd4ca3e6a 100644 --- a/test/e2e/e2eutil.go +++ b/test/e2e/e2eutil.go @@ -4,23 +4,15 @@ import ( "context" "fmt" "reflect" - "testing" - "time" "github.com/mongodb/mongodb-kubernetes-operator/pkg/util/envvar" - "github.com/pkg/errors" - mdbv1 "github.com/mongodb/mongodb-kubernetes-operator/api/v1" - "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/statefulset" - appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" apiErrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/wait" - "sigs.k8s.io/controller-runtime/pkg/client" k8sClient "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -47,127 +39,6 @@ func UpdateMongoDBResource(original *mdbv1.MongoDBCommunity, updateFunc func(*md return TestClient.Update(context.TODO(), original) } -// WaitForConfigMapToExist waits until a ConfigMap of the given name exists -// using the provided retryInterval and timeout -func WaitForConfigMapToExist(cmName string, retryInterval, timeout time.Duration) (corev1.ConfigMap, error) { - cm := corev1.ConfigMap{} - return cm, waitForRuntimeObjectToExist(cmName, retryInterval, timeout, &cm, OperatorNamespace) -} - -// WaitForSecretToExist waits until a Secret of the given name exists -// using the provided retryInterval and timeout -func WaitForSecretToExist(cmName string, retryInterval, timeout time.Duration, namespace string) (corev1.Secret, error) { - s := corev1.Secret{} - return s, waitForRuntimeObjectToExist(cmName, retryInterval, timeout, &s, namespace) -} - -// WaitForMongoDBToReachPhase waits until the given MongoDB resource reaches the expected phase -func WaitForMongoDBToReachPhase(t *testing.T, mdb *mdbv1.MongoDBCommunity, phase mdbv1.Phase, retryInterval, timeout time.Duration) error { - return waitForMongoDBCondition(mdb, retryInterval, timeout, func(db mdbv1.MongoDBCommunity) bool { - t.Logf("current phase: %s, waiting for phase: %s", db.Status.Phase, phase) - return db.Status.Phase == phase - }) -} - -// waitForMongoDBCondition polls and waits for a given condition to be true -func waitForMongoDBCondition(mdb *mdbv1.MongoDBCommunity, retryInterval, timeout time.Duration, condition func(mdbv1.MongoDBCommunity) bool) error { - mdbNew := mdbv1.MongoDBCommunity{} - return wait.Poll(retryInterval, timeout, func() (done bool, err error) { - err = TestClient.Get(context.TODO(), mdb.NamespacedName(), &mdbNew) - if err != nil { - return false, err - } - ready := condition(mdbNew) - return ready, nil - }) -} - -// WaitForStatefulSetToExist waits until a StatefulSet of the given name exists -// using the provided retryInterval and timeout -func WaitForStatefulSetToExist(stsName string, retryInterval, timeout time.Duration, namespace string) (appsv1.StatefulSet, error) { - sts := appsv1.StatefulSet{} - return sts, waitForRuntimeObjectToExist(stsName, retryInterval, timeout, &sts, namespace) -} - -// WaitForStatefulSetToHaveUpdateStrategy waits until all replicas of the StatefulSet with the given name -// have reached the ready status -func WaitForStatefulSetToHaveUpdateStrategy(t *testing.T, mdb *mdbv1.MongoDBCommunity, strategy appsv1.StatefulSetUpdateStrategyType, retryInterval, timeout time.Duration) error { - return waitForStatefulSetCondition(t, mdb, retryInterval, timeout, func(sts appsv1.StatefulSet) bool { - return sts.Spec.UpdateStrategy.Type == strategy - }) -} - -// WaitForStatefulSetToBeReady waits until all replicas of the StatefulSet with the given name -// have reached the ready status -func WaitForStatefulSetToBeReady(t *testing.T, mdb *mdbv1.MongoDBCommunity, retryInterval, timeout time.Duration) error { - return waitForStatefulSetCondition(t, mdb, retryInterval, timeout, func(sts appsv1.StatefulSet) bool { - return statefulset.IsReady(sts, mdb.Spec.Members) - }) -} - -// waitForStatefulSetCondition waits until all replicas of the StatefulSet with the given name -// is not ready. -func WaitForStatefulSetToBeUnready(t *testing.T, mdb *mdbv1.MongoDBCommunity, retryInterval, timeout time.Duration) error { - return waitForStatefulSetCondition(t, mdb, retryInterval, timeout, func(sts appsv1.StatefulSet) bool { - return !statefulset.IsReady(sts, mdb.Spec.Members) - }) -} - -// WaitForStatefulSetToBeReadyAfterScaleDown waits for just the ready replicas to be correct -// and does not account for the updated replicas -func WaitForStatefulSetToBeReadyAfterScaleDown(t *testing.T, mdb *mdbv1.MongoDBCommunity, retryInterval, timeout time.Duration) error { - return waitForStatefulSetCondition(t, mdb, retryInterval, timeout, func(sts appsv1.StatefulSet) bool { - return int32(mdb.Spec.Members) == sts.Status.ReadyReplicas - }) -} - -func waitForStatefulSetCondition(t *testing.T, mdb *mdbv1.MongoDBCommunity, retryInterval, timeout time.Duration, condition func(set appsv1.StatefulSet) bool) error { - _, err := WaitForStatefulSetToExist(mdb.Name, retryInterval, timeout, mdb.Namespace) - if err != nil { - return errors.Errorf("error waiting for stateful set to be created: %s", err) - } - - sts := appsv1.StatefulSet{} - return wait.Poll(retryInterval, timeout, func() (done bool, err error) { - err = TestClient.Get(context.TODO(), mdb.NamespacedName(), &sts) - if err != nil { - return false, err - } - t.Logf("Waiting for %s to have %d replicas. Current ready replicas: %d, Current updated replicas: %d, Current generation: %d, Observed Generation: %d\n", - mdb.Name, mdb.Spec.Members, sts.Status.ReadyReplicas, sts.Status.UpdatedReplicas, sts.Generation, sts.Status.ObservedGeneration) - ready := condition(sts) - return ready, nil - }) -} - -func WaitForPodReadiness(t *testing.T, isReady bool, containerName string, timeout time.Duration, pod corev1.Pod) error { - return wait.Poll(time.Second*3, timeout, func() (done bool, err error) { - err = TestClient.Get(context.TODO(), types.NamespacedName{Name: pod.Name, Namespace: pod.Namespace}, &pod) - if err != nil { - return false, err - } - for _, status := range pod.Status.ContainerStatuses { - t.Logf("%s (%s), ready: %v\n", pod.Name, status.Name, status.Ready) - if status.Name == containerName && status.Ready == isReady { - return true, nil - } - } - return false, nil - }) -} - -// waitForRuntimeObjectToExist waits until a runtime.Object of the given name exists -// using the provided retryInterval and timeout provided. -func waitForRuntimeObjectToExist(name string, retryInterval, timeout time.Duration, obj client.Object, namespace string) error { - return wait.Poll(retryInterval, timeout, func() (done bool, err error) { - err = TestClient.Get(context.TODO(), types.NamespacedName{Name: name, Namespace: namespace}, obj) - if err != nil { - return false, client.IgnoreNotFound(err) - } - return true, nil - }) -} - func NewTestMongoDB(ctx *Context, name string, namespace string) (mdbv1.MongoDBCommunity, mdbv1.MongoDBUser) { mongodbNamespace := namespace if mongodbNamespace == "" { diff --git a/test/e2e/mongodbtests/mongodbtests.go b/test/e2e/mongodbtests/mongodbtests.go index 6dce668c3..94acc2eee 100644 --- a/test/e2e/mongodbtests/mongodbtests.go +++ b/test/e2e/mongodbtests/mongodbtests.go @@ -4,10 +4,12 @@ import ( "context" "encoding/json" "fmt" - "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/container" "testing" "time" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/container" + "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/util/wait" + mdbv1 "github.com/mongodb/mongodb-kubernetes-operator/api/v1" "github.com/mongodb/mongodb-kubernetes-operator/pkg/authentication/scram" "github.com/mongodb/mongodb-kubernetes-operator/pkg/automationconfig" @@ -46,7 +48,7 @@ func StatefulSetBecomesUnready(mdb *mdbv1.MongoDBCommunity) func(t *testing.T) { // failure threshold being high func StatefulSetIsReadyAfterScaleDown(mdb *mdbv1.MongoDBCommunity) func(t *testing.T) { return func(t *testing.T) { - err := e2eutil.WaitForStatefulSetToBeReadyAfterScaleDown(t, mdb, time.Second*60, time.Minute*45) + err := wait.ForStatefulSetToBeReadyAfterScaleDown(t, mdb, time.Second*60, time.Minute*45) if err != nil { t.Fatal(err) } @@ -58,7 +60,7 @@ func StatefulSetIsReadyAfterScaleDown(mdb *mdbv1.MongoDBCommunity) func(t *testi // reaches the running state func statefulSetIsReady(mdb *mdbv1.MongoDBCommunity, interval time.Duration, timeout time.Duration) func(t *testing.T) { return func(t *testing.T) { - err := e2eutil.WaitForStatefulSetToBeReady(t, mdb, interval, timeout) + err := wait.ForStatefulSetToBeReady(t, mdb, interval, timeout) if err != nil { t.Fatal(err) } @@ -69,7 +71,7 @@ func statefulSetIsReady(mdb *mdbv1.MongoDBCommunity, interval time.Duration, tim // statefulSetIsNotReady ensures that the underlying stateful set reaches the unready state. func statefulSetIsNotReady(mdb *mdbv1.MongoDBCommunity, interval time.Duration, timeout time.Duration) func(t *testing.T) { return func(t *testing.T) { - err := e2eutil.WaitForStatefulSetToBeUnready(t, mdb, interval, timeout) + err := wait.ForStatefulSetToBeUnready(t, mdb, interval, timeout) if err != nil { t.Fatal(err) } @@ -136,7 +138,7 @@ func ConnectionStringSecretsAreConfigured(mdb *mdbv1.MongoDBCommunity, expectedO // resource has the correct Update Strategy func StatefulSetHasUpdateStrategy(mdb *mdbv1.MongoDBCommunity, strategy appsv1.StatefulSetUpdateStrategyType) func(t *testing.T) { return func(t *testing.T) { - err := e2eutil.WaitForStatefulSetToHaveUpdateStrategy(t, mdb, strategy, time.Second*15, time.Minute*8) + err := wait.ForStatefulSetToHaveUpdateStrategy(t, mdb, strategy, time.Second*15, time.Minute*8) if err != nil { t.Fatal(err) } @@ -147,7 +149,7 @@ func StatefulSetHasUpdateStrategy(mdb *mdbv1.MongoDBCommunity, strategy appsv1.S // MongoDBReachesRunningPhase ensure the MongoDB resource reaches the Running phase func MongoDBReachesRunningPhase(mdb *mdbv1.MongoDBCommunity) func(t *testing.T) { return func(t *testing.T) { - err := e2eutil.WaitForMongoDBToReachPhase(t, mdb, mdbv1.Running, time.Second*15, time.Minute*12) + err := wait.ForMongoDBToReachPhase(t, mdb, mdbv1.Running, time.Second*15, time.Minute*12) if err != nil { t.Fatal(err) } @@ -158,7 +160,7 @@ func MongoDBReachesRunningPhase(mdb *mdbv1.MongoDBCommunity) func(t *testing.T) // MongoDBReachesFailed ensure the MongoDB resource reaches the Failed phase. func MongoDBReachesFailedPhase(mdb *mdbv1.MongoDBCommunity) func(t *testing.T) { return func(t *testing.T) { - err := e2eutil.WaitForMongoDBToReachPhase(t, mdb, mdbv1.Failed, time.Second*15, time.Minute*5) + err := wait.ForMongoDBToReachPhase(t, mdb, mdbv1.Failed, time.Second*15, time.Minute*5) if err != nil { t.Fatal(err) } @@ -168,7 +170,7 @@ func MongoDBReachesFailedPhase(mdb *mdbv1.MongoDBCommunity) func(t *testing.T) { func AutomationConfigSecretExists(mdb *mdbv1.MongoDBCommunity) func(t *testing.T) { return func(t *testing.T) { - s, err := e2eutil.WaitForSecretToExist(mdb.AutomationConfigSecretName(), time.Second*5, time.Minute*1, mdb.Namespace) + s, err := wait.ForSecretToExist(mdb.AutomationConfigSecretName(), time.Second*5, time.Minute*1, mdb.Namespace) assert.NoError(t, err) t.Logf("Secret %s/%s was successfully created", mdb.AutomationConfigSecretName(), mdb.Namespace) @@ -218,7 +220,7 @@ func CreateMongoDBResource(mdb *mdbv1.MongoDBCommunity, ctx *e2eutil.Context) fu func GetConnectionStringSecret(mdb mdbv1.MongoDBCommunity, user scram.User) corev1.Secret { secret := corev1.Secret{} secretNamespacedName := types.NamespacedName{Name: user.GetConnectionStringSecretName(mdb), Namespace: mdb.Namespace} - e2eutil.TestClient.Get(context.TODO(), secretNamespacedName, &secret) + _ = e2eutil.TestClient.Get(context.TODO(), secretNamespacedName, &secret) return secret } @@ -353,7 +355,7 @@ func StatefulSetContainerConditionIsTrue(mdb *mdbv1.MongoDBCommunity, containerN func PodContainerBecomesNotReady(mdb *mdbv1.MongoDBCommunity, podNum int, containerName string) func(*testing.T) { return func(t *testing.T) { pod := podFromMongoDBCommunity(mdb, podNum) - assert.NoError(t, e2eutil.WaitForPodReadiness(t, false, containerName, time.Minute*10, pod)) + assert.NoError(t, wait.ForPodReadiness(t, false, containerName, time.Minute*10, pod)) } } @@ -361,7 +363,7 @@ func PodContainerBecomesNotReady(mdb *mdbv1.MongoDBCommunity, podNum int, contai func PodContainerBecomesReady(mdb *mdbv1.MongoDBCommunity, podNum int, containerName string) func(*testing.T) { return func(t *testing.T) { pod := podFromMongoDBCommunity(mdb, podNum) - assert.NoError(t, e2eutil.WaitForPodReadiness(t, true, containerName, time.Minute*3, pod)) + assert.NoError(t, wait.ForPodReadiness(t, true, containerName, time.Minute*3, pod)) } } diff --git a/test/e2e/util/wait/wait.go b/test/e2e/util/wait/wait.go new file mode 100644 index 000000000..37d3807d3 --- /dev/null +++ b/test/e2e/util/wait/wait.go @@ -0,0 +1,138 @@ +package wait + +import ( + "context" + "testing" + "time" + + mdbv1 "github.com/mongodb/mongodb-kubernetes-operator/api/v1" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/statefulset" + e2eutil "github.com/mongodb/mongodb-kubernetes-operator/test/e2e" + "github.com/pkg/errors" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// ForConfigMapToExist waits until a ConfigMap of the given name exists +// using the provided retryInterval and timeout +func ForConfigMapToExist(cmName string, retryInterval, timeout time.Duration) (corev1.ConfigMap, error) { + cm := corev1.ConfigMap{} + return cm, waitForRuntimeObjectToExist(cmName, retryInterval, timeout, &cm, e2eutil.OperatorNamespace) +} + +// ForSecretToExist waits until a Secret of the given name exists +// using the provided retryInterval and timeout +func ForSecretToExist(cmName string, retryInterval, timeout time.Duration, namespace string) (corev1.Secret, error) { + s := corev1.Secret{} + return s, waitForRuntimeObjectToExist(cmName, retryInterval, timeout, &s, namespace) +} + +// ForMongoDBToReachPhase waits until the given MongoDB resource reaches the expected phase +func ForMongoDBToReachPhase(t *testing.T, mdb *mdbv1.MongoDBCommunity, phase mdbv1.Phase, retryInterval, timeout time.Duration) error { + return waitForMongoDBCondition(mdb, retryInterval, timeout, func(db mdbv1.MongoDBCommunity) bool { + t.Logf("current phase: %s, waiting for phase: %s", db.Status.Phase, phase) + return db.Status.Phase == phase + }) +} + +// waitForMongoDBCondition polls and waits for a given condition to be true +func waitForMongoDBCondition(mdb *mdbv1.MongoDBCommunity, retryInterval, timeout time.Duration, condition func(mdbv1.MongoDBCommunity) bool) error { + mdbNew := mdbv1.MongoDBCommunity{} + return wait.Poll(retryInterval, timeout, func() (done bool, err error) { + err = e2eutil.TestClient.Get(context.TODO(), mdb.NamespacedName(), &mdbNew) + if err != nil { + return false, err + } + ready := condition(mdbNew) + return ready, nil + }) +} + +// ForStatefulSetToExist waits until a StatefulSet of the given name exists +// using the provided retryInterval and timeout +func ForStatefulSetToExist(stsName string, retryInterval, timeout time.Duration, namespace string) (appsv1.StatefulSet, error) { + sts := appsv1.StatefulSet{} + return sts, waitForRuntimeObjectToExist(stsName, retryInterval, timeout, &sts, namespace) +} + +// ForStatefulSetToHaveUpdateStrategy waits until all replicas of the StatefulSet with the given name +// have reached the ready status +func ForStatefulSetToHaveUpdateStrategy(t *testing.T, mdb *mdbv1.MongoDBCommunity, strategy appsv1.StatefulSetUpdateStrategyType, retryInterval, timeout time.Duration) error { + return waitForStatefulSetCondition(t, mdb, retryInterval, timeout, func(sts appsv1.StatefulSet) bool { + return sts.Spec.UpdateStrategy.Type == strategy + }) +} + +// ForStatefulSetToBeReady waits until all replicas of the StatefulSet with the given name +// have reached the ready status +func ForStatefulSetToBeReady(t *testing.T, mdb *mdbv1.MongoDBCommunity, retryInterval, timeout time.Duration) error { + return waitForStatefulSetCondition(t, mdb, retryInterval, timeout, func(sts appsv1.StatefulSet) bool { + return statefulset.IsReady(sts, mdb.Spec.Members) + }) +} + +// ForStatefulSetToBeUnready waits until all replicas of the StatefulSet with the given name +// is not ready. +func ForStatefulSetToBeUnready(t *testing.T, mdb *mdbv1.MongoDBCommunity, retryInterval, timeout time.Duration) error { + return waitForStatefulSetCondition(t, mdb, retryInterval, timeout, func(sts appsv1.StatefulSet) bool { + return !statefulset.IsReady(sts, mdb.Spec.Members) + }) +} + +// ForStatefulSetToBeReadyAfterScaleDown waits for just the ready replicas to be correct +// and does not account for the updated replicas +func ForStatefulSetToBeReadyAfterScaleDown(t *testing.T, mdb *mdbv1.MongoDBCommunity, retryInterval, timeout time.Duration) error { + return waitForStatefulSetCondition(t, mdb, retryInterval, timeout, func(sts appsv1.StatefulSet) bool { + return int32(mdb.Spec.Members) == sts.Status.ReadyReplicas + }) +} + +func waitForStatefulSetCondition(t *testing.T, mdb *mdbv1.MongoDBCommunity, retryInterval, timeout time.Duration, condition func(set appsv1.StatefulSet) bool) error { + _, err := ForStatefulSetToExist(mdb.Name, retryInterval, timeout, mdb.Namespace) + if err != nil { + return errors.Errorf("error waiting for stateful set to be created: %s", err) + } + + sts := appsv1.StatefulSet{} + return wait.Poll(retryInterval, timeout, func() (done bool, err error) { + err = e2eutil.TestClient.Get(context.TODO(), mdb.NamespacedName(), &sts) + if err != nil { + return false, err + } + t.Logf("Waiting for %s to have %d replicas. Current ready replicas: %d, Current updated replicas: %d, Current generation: %d, Observed Generation: %d\n", + mdb.Name, mdb.Spec.Members, sts.Status.ReadyReplicas, sts.Status.UpdatedReplicas, sts.Generation, sts.Status.ObservedGeneration) + ready := condition(sts) + return ready, nil + }) +} + +func ForPodReadiness(t *testing.T, isReady bool, containerName string, timeout time.Duration, pod corev1.Pod) error { + return wait.Poll(time.Second*3, timeout, func() (done bool, err error) { + err = e2eutil.TestClient.Get(context.TODO(), types.NamespacedName{Name: pod.Name, Namespace: pod.Namespace}, &pod) + if err != nil { + return false, err + } + for _, status := range pod.Status.ContainerStatuses { + t.Logf("%s (%s), ready: %v\n", pod.Name, status.Name, status.Ready) + if status.Name == containerName && status.Ready == isReady { + return true, nil + } + } + return false, nil + }) +} + +// waitForRuntimeObjectToExist waits until a runtime.Object of the given name exists +// using the provided retryInterval and timeout provided. +func waitForRuntimeObjectToExist(name string, retryInterval, timeout time.Duration, obj client.Object, namespace string) error { + return wait.Poll(retryInterval, timeout, func() (done bool, err error) { + err = e2eutil.TestClient.Get(context.TODO(), types.NamespacedName{Name: name, Namespace: namespace}, obj) + if err != nil { + return false, client.IgnoreNotFound(err) + } + return true, nil + }) +}