Skip to content

Increase Stability of FCV tests #550

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Jun 10, 2021
Merged

Increase Stability of FCV tests #550

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
dc64305
ignore error
chatton Jun 9, 2021
3444143
Make timeout and retry interval configurable
chatton Jun 10, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion api/v1/mongodbcommunity_types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -339,7 +339,7 @@ type Authentication struct {
// +optional
// +kubebuilder:default:=true
// +nullable
IgnoreUnknownUsers *bool `json:"ignoreUnknownUsers"`
IgnoreUnknownUsers *bool `json:"ignoreUnknownUsers,omitempty"`
Copy link
Contributor

@rodrigovalin rodrigovalin Jun 10, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

is this related to fcv?

Copy link
Contributor Author

@chatton chatton Jun 10, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

no sorry, this was an unrelated fix.

}

// +kubebuilder:validation:Enum=SCRAM
Expand Down
6 changes: 4 additions & 2 deletions test/e2e/feature_compatibility_version/feature_compatibility_version_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ import (
"testing"
"time"

"github.com/mongodb/mongodb-kubernetes-operator/test/e2e/util/wait"

. "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/util/mongotester"

e2eutil "github.com/mongodb/mongodb-kubernetes-operator/test/e2e"
Expand Down Expand Up @@ -49,7 +51,7 @@ func TestFeatureCompatibilityVersion(t *testing.T) {
t.Run("MongoDB is reachable while version is upgraded", func(t *testing.T) {
defer tester.StartBackgroundConnectivityTest(t, time.Second*10)()
t.Run("Test Version can be upgraded", mongodbtests.ChangeVersion(&mdb, "4.2.6"))
t.Run("Stateful Set Reaches Ready State, after Upgrading", mongodbtests.StatefulSetBecomesReady(&mdb))
t.Run("Stateful Set Reaches Ready State, after Upgrading", mongodbtests.StatefulSetBecomesReady(&mdb, wait.Timeout(20*time.Minute)))
})

t.Run("Test Basic Connectivity after upgrade has completed", tester.ConnectivitySucceeds())
Expand All @@ -59,7 +61,7 @@ func TestFeatureCompatibilityVersion(t *testing.T) {
t.Run("MongoDB is reachable while version is downgraded", func(t *testing.T) {
defer tester.StartBackgroundConnectivityTest(t, time.Second*10)()
t.Run("Test Version can be downgraded", mongodbtests.ChangeVersion(&mdb, "4.0.6"))
t.Run("Stateful Set Reaches Ready State, after Upgrading", mongodbtests.StatefulSetBecomesReady(&mdb))
t.Run("Stateful Set Reaches Ready State, after Upgrading", mongodbtests.StatefulSetBecomesReady(&mdb, wait.Timeout(20*time.Minute)))
})

t.Run("Test FeatureCompatibilityVersion, after downgrade, is 4.0", tester.HasFCV("4.0", 3))
Expand Down
6 changes: 4 additions & 2 deletions test/e2e/feature_compatibility_version_upgrade/feature_compatibility_version_upgrade_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ import (
"testing"
"time"

"github.com/mongodb/mongodb-kubernetes-operator/test/e2e/util/wait"

"github.com/mongodb/mongodb-kubernetes-operator/test/e2e/util/mongotester"

mdbv1 "github.com/mongodb/mongodb-kubernetes-operator/api/v1"
Expand Down Expand Up @@ -51,7 +53,7 @@ func TestFeatureCompatibilityVersionUpgrade(t *testing.T) {
t.Run("MongoDB is reachable", func(t *testing.T) {
defer tester.StartBackgroundConnectivityTest(t, time.Second*10)()
t.Run("Test Version can be upgraded", mongodbtests.ChangeVersion(&mdb, "4.2.6"))
t.Run("Stateful Set Reaches Ready State, after Upgrading", mongodbtests.StatefulSetBecomesReady(&mdb))
t.Run("Stateful Set Reaches Ready State, after Upgrading", mongodbtests.StatefulSetBecomesReady(&mdb, wait.Timeout(20*time.Minute)))
t.Run("Test Basic Connectivity after upgrade has completed", tester.ConnectivitySucceeds())
})

Expand All @@ -64,7 +66,7 @@ func TestFeatureCompatibilityVersionUpgrade(t *testing.T) {
})
assert.NoError(t, err)
})
t.Run("Stateful Set Reaches Ready State", mongodbtests.StatefulSetBecomesReady(&mdb))
t.Run("Stateful Set Reaches Ready State", mongodbtests.StatefulSetBecomesReady(&mdb, wait.Timeout(20*time.Minute)))
t.Run("MongoDB Reaches Running Phase", mongodbtests.MongoDBReachesRunningPhase(&mdb))
})
t.Run("Test FeatureCompatibilityVersion, after upgrade, is 4.2", tester.HasFCV("4.2", 3))
Expand Down
30 changes: 20 additions & 10 deletions test/e2e/mongodbtests/mongodbtests.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,22 +33,32 @@ func SkipTestIfLocal(t *testing.T, msg string, f func(t *testing.T)) {

// StatefulSetBecomesReady ensures that the underlying stateful set
// reaches the running state.
func StatefulSetBecomesReady(mdb *mdbv1.MongoDBCommunity) func(t *testing.T) {
return statefulSetIsReady(mdb, time.Second*15, time.Minute*12)
func StatefulSetBecomesReady(mdb *mdbv1.MongoDBCommunity, opts ...wait.Configuration) func(t *testing.T) {
defaultOpts := []wait.Configuration{
wait.RetryInterval(time.Second * 15),
wait.Timeout(time.Minute * 15),
}
defaultOpts = append(defaultOpts, opts...)
return statefulSetIsReady(mdb, defaultOpts...)
}

// StatefulSetBecomesUnready ensures the underlying stateful set reaches
// the unready state.
func StatefulSetBecomesUnready(mdb *mdbv1.MongoDBCommunity) func(t *testing.T) {
return statefulSetIsNotReady(mdb, time.Second*15, time.Minute*12)
func StatefulSetBecomesUnready(mdb *mdbv1.MongoDBCommunity, opts ...wait.Configuration) func(t *testing.T) {
defaultOpts := []wait.Configuration{
wait.RetryInterval(time.Second * 15),
wait.Timeout(time.Minute * 15),
}
defaultOpts = append(defaultOpts, opts...)
return statefulSetIsNotReady(mdb, defaultOpts...)
}

// StatefulSetIsReadyAfterScaleDown ensures that a replica set is scaled down correctly
// note: scaling down takes considerably longer than scaling up due the readiness probe
// failure threshold being high
func StatefulSetIsReadyAfterScaleDown(mdb *mdbv1.MongoDBCommunity) func(t *testing.T) {
return func(t *testing.T) {
err := wait.ForStatefulSetToBeReadyAfterScaleDown(t, mdb, time.Second*60, time.Minute*45)
err := wait.ForStatefulSetToBeReadyAfterScaleDown(t, mdb, wait.RetryInterval(time.Second*60), wait.Timeout(time.Minute*45))
if err != nil {
t.Fatal(err)
}
Expand All @@ -58,9 +68,9 @@ func StatefulSetIsReadyAfterScaleDown(mdb *mdbv1.MongoDBCommunity) func(t *testi

// StatefulSetIsReady ensures that the underlying stateful set
// reaches the running state
func statefulSetIsReady(mdb *mdbv1.MongoDBCommunity, interval time.Duration, timeout time.Duration) func(t *testing.T) {
func statefulSetIsReady(mdb *mdbv1.MongoDBCommunity, opts ...wait.Configuration) func(t *testing.T) {
return func(t *testing.T) {
err := wait.ForStatefulSetToBeReady(t, mdb, interval, timeout)
err := wait.ForStatefulSetToBeReady(t, mdb, opts...)
if err != nil {
t.Fatal(err)
}
Expand All @@ -69,9 +79,9 @@ func statefulSetIsReady(mdb *mdbv1.MongoDBCommunity, interval time.Duration, tim
}

// statefulSetIsNotReady ensures that the underlying stateful set reaches the unready state.
func statefulSetIsNotReady(mdb *mdbv1.MongoDBCommunity, interval time.Duration, timeout time.Duration) func(t *testing.T) {
func statefulSetIsNotReady(mdb *mdbv1.MongoDBCommunity, opts ...wait.Configuration) func(t *testing.T) {
return func(t *testing.T) {
err := wait.ForStatefulSetToBeUnready(t, mdb, interval, timeout)
err := wait.ForStatefulSetToBeUnready(t, mdb, opts...)
if err != nil {
t.Fatal(err)
}
Expand Down Expand Up @@ -138,7 +148,7 @@ func ConnectionStringSecretsAreConfigured(mdb *mdbv1.MongoDBCommunity, expectedO
// resource has the correct Update Strategy
func StatefulSetHasUpdateStrategy(mdb *mdbv1.MongoDBCommunity, strategy appsv1.StatefulSetUpdateStrategyType) func(t *testing.T) {
return func(t *testing.T) {
err := wait.ForStatefulSetToHaveUpdateStrategy(t, mdb, strategy, time.Second*15, time.Minute*8)
err := wait.ForStatefulSetToHaveUpdateStrategy(t, mdb, strategy, wait.RetryInterval(time.Second*15), wait.Timeout(time.Minute*8))
if err != nil {
t.Fatal(err)
}
Expand Down
26 changes: 15 additions & 11 deletions test/e2e/util/wait/wait.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,44 +60,48 @@ func ForStatefulSetToExist(stsName string, retryInterval, timeout time.Duration,

// ForStatefulSetToHaveUpdateStrategy waits until all replicas of the StatefulSet with the given name
// have reached the ready status
func ForStatefulSetToHaveUpdateStrategy(t *testing.T, mdb *mdbv1.MongoDBCommunity, strategy appsv1.StatefulSetUpdateStrategyType, retryInterval, timeout time.Duration) error {
return waitForStatefulSetCondition(t, mdb, retryInterval, timeout, func(sts appsv1.StatefulSet) bool {
func ForStatefulSetToHaveUpdateStrategy(t *testing.T, mdb *mdbv1.MongoDBCommunity, strategy appsv1.StatefulSetUpdateStrategyType, opts ...Configuration) error {
options := newOptions(opts...)
return waitForStatefulSetCondition(t, mdb, options, func(sts appsv1.StatefulSet) bool {
return sts.Spec.UpdateStrategy.Type == strategy
})
}

// ForStatefulSetToBeReady waits until all replicas of the StatefulSet with the given name
// have reached the ready status
func ForStatefulSetToBeReady(t *testing.T, mdb *mdbv1.MongoDBCommunity, retryInterval, timeout time.Duration) error {
return waitForStatefulSetCondition(t, mdb, retryInterval, timeout, func(sts appsv1.StatefulSet) bool {
func ForStatefulSetToBeReady(t *testing.T, mdb *mdbv1.MongoDBCommunity, opts ...Configuration) error {
options := newOptions(opts...)
return waitForStatefulSetCondition(t, mdb, options, func(sts appsv1.StatefulSet) bool {
return statefulset.IsReady(sts, mdb.Spec.Members)
})
}

// ForStatefulSetToBeUnready waits until all replicas of the StatefulSet with the given name
// is not ready.
func ForStatefulSetToBeUnready(t *testing.T, mdb *mdbv1.MongoDBCommunity, retryInterval, timeout time.Duration) error {
return waitForStatefulSetCondition(t, mdb, retryInterval, timeout, func(sts appsv1.StatefulSet) bool {
func ForStatefulSetToBeUnready(t *testing.T, mdb *mdbv1.MongoDBCommunity, opts ...Configuration) error {
options := newOptions(opts...)
return waitForStatefulSetCondition(t, mdb, options, func(sts appsv1.StatefulSet) bool {
return !statefulset.IsReady(sts, mdb.Spec.Members)
})
}

// ForStatefulSetToBeReadyAfterScaleDown waits for just the ready replicas to be correct
// and does not account for the updated replicas
func ForStatefulSetToBeReadyAfterScaleDown(t *testing.T, mdb *mdbv1.MongoDBCommunity, retryInterval, timeout time.Duration) error {
return waitForStatefulSetCondition(t, mdb, retryInterval, timeout, func(sts appsv1.StatefulSet) bool {
func ForStatefulSetToBeReadyAfterScaleDown(t *testing.T, mdb *mdbv1.MongoDBCommunity, opts ...Configuration) error {
options := newOptions(opts...)
return waitForStatefulSetCondition(t, mdb, options, func(sts appsv1.StatefulSet) bool {
return int32(mdb.Spec.Members) == sts.Status.ReadyReplicas
})
}

func waitForStatefulSetCondition(t *testing.T, mdb *mdbv1.MongoDBCommunity, retryInterval, timeout time.Duration, condition func(set appsv1.StatefulSet) bool) error {
_, err := ForStatefulSetToExist(mdb.Name, retryInterval, timeout, mdb.Namespace)
func waitForStatefulSetCondition(t *testing.T, mdb *mdbv1.MongoDBCommunity, waitOpts Options, condition func(set appsv1.StatefulSet) bool) error {
_, err := ForStatefulSetToExist(mdb.Name, waitOpts.RetryInterval, waitOpts.Timeout, mdb.Namespace)
if err != nil {
return errors.Errorf("error waiting for stateful set to be created: %s", err)
}

sts := appsv1.StatefulSet{}
return wait.Poll(retryInterval, timeout, func() (done bool, err error) {
return wait.Poll(waitOpts.RetryInterval, waitOpts.Timeout, func() (done bool, err error) {
err = e2eutil.TestClient.Get(context.TODO(), mdb.NamespacedName(), &sts)
if err != nil {
return false, err
Expand Down
41 changes: 41 additions & 0 deletions test/e2e/util/wait/wait_options.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
package wait

import "time"

type Configuration func(*Options)

// Options holds values which can be configured when waiting for specific confitions.
type Options struct {
RetryInterval time.Duration
Timeout time.Duration
}

// RetryInterval specifies the RetryInterval
func RetryInterval(retryInterval time.Duration) Configuration {
return func(options *Options) {
options.RetryInterval = retryInterval
}
}

// Timeout specifies the Timeout
func Timeout(timeout time.Duration) Configuration {
return func(options *Options) {
options.Timeout = timeout
}
}

// newOptions returns an Options that has been configured with default values.
func newOptions(fns ...Configuration) Options {
defaults := defaultStatefulSetReadinessOptions()
for _, fn := range fns {
fn(&defaults)
}
return defaults
}

func defaultStatefulSetReadinessOptions() Options {
return Options{
RetryInterval: time.Second * 15,
Timeout: time.Minute * 12,
}
}