diff --git a/.evergreen-tasks.yml b/.evergreen-tasks.yml index 8fb63f71c..2d0803097 100644 --- a/.evergreen-tasks.yml +++ b/.evergreen-tasks.yml @@ -1116,7 +1116,6 @@ tasks: - name: e2e_multi_cluster_validation tags: [ "patch-run" ] - exec_timeout_secs: 1000 commands: - func: e2e_test diff --git a/docker/mongodb-kubernetes-tests/tests/multicluster_appdb/multicluster_appdb_disaster_recovery.py b/docker/mongodb-kubernetes-tests/tests/multicluster_appdb/multicluster_appdb_disaster_recovery.py index ece7ea32c..b0208b518 100644 --- a/docker/mongodb-kubernetes-tests/tests/multicluster_appdb/multicluster_appdb_disaster_recovery.py +++ b/docker/mongodb-kubernetes-tests/tests/multicluster_appdb/multicluster_appdb_disaster_recovery.py @@ -152,11 +152,12 @@ def test_delete_om_and_appdb_statefulset_in_failed_cluster( # delete OM to simulate losing Ops Manager application # this is only for testing unavailability of the OM application, it's not testing losing OM cluster # we don't delete here any additional resources (secrets, configmaps) that are required for a proper OM recovery testing + # it will be immediately recreated by the operator, so we cannot check if it was deleted delete_statefulset( ops_manager.namespace, ops_manager.name, propagation_policy="Background", - api_client=central_cluster_client, + api_client=get_member_cluster_api_client(OM_MEMBER_CLUSTER_NAME), ) except kubernetes.client.ApiException as e: if e.status != 404: @@ -184,14 +185,6 @@ def statefulset_is_deleted(namespace: str, name: str, api_client=Optional[kubern else: raise e - run_periodically( - lambda: statefulset_is_deleted( - ops_manager.namespace, - ops_manager.name, - api_client=get_member_cluster_api_client(OM_MEMBER_CLUSTER_NAME), - ), - timeout=120, - ) run_periodically( lambda: statefulset_is_deleted( ops_manager.namespace, diff --git a/scripts/funcs/multicluster b/scripts/funcs/multicluster index ef0cb9b1f..b5b1e5e59 100644 --- a/scripts/funcs/multicluster +++ b/scripts/funcs/multicluster @@ -128,26 +128,6 @@ EOF sleep 1 local service_account_name="operator-tests-multi-cluster-service-account" - - local secret_name - secret_name="$(kubectl --context "${CENTRAL_CLUSTER}" get secret -n "${NAMESPACE}" | { grep "${service_account_name}" || test $? = 1; } | awk '{ print $1 }')" - if [[ "${secret_name}" == "" ]]; then - secret_name="${service_account_name}-token-secret" - create_service_account_token_secret "${CENTRAL_CLUSTER}" "${service_account_name}" "${secret_name}" - fi - - local central_cluster_token - central_cluster_token="$(kubectl --context "${CENTRAL_CLUSTER}" get secret "${secret_name}" -o jsonpath='{ .data.token}' -n "${NAMESPACE}" | base64 -d)" - echo "Creating Multi Cluster configuration secret" - - configuration_params=( - "--from-literal=central_cluster=${CENTRAL_CLUSTER}" - ) - - configuration_params+=( - "--from-literal=${CENTRAL_CLUSTER}=${central_cluster_token}" - ) - local secret_name secret_name="$(kubectl --context "${CENTRAL_CLUSTER}" get secret -n "${NAMESPACE}" | { grep "${service_account_name}" || test $? = 1; } | awk '{ print $1 }')" if [[ "${secret_name}" == "" ]]; then @@ -175,7 +155,18 @@ EOF create_service_account_token_secret "${member_cluster}" "${service_account_name}" "${secret_name}" fi - member_cluster_token="$(kubectl --context "${member_cluster}" get secret "${secret_name}" -o jsonpath='{ .data.token}' -n "${NAMESPACE}" | base64 -d)" + # Retry up to 10 times if .data.token is not yet populated + for _ in {1..10}; do + member_cluster_token="$(kubectl --context "${member_cluster}" get secret "${secret_name}" -o jsonpath='{ .data.token }' -n "${NAMESPACE}" | base64 -d)" + if [[ -n "${member_cluster_token}" ]]; then + break + fi + sleep 1 + done + if [[ -z "${member_cluster_token}" ]]; then + echo "Error: .data.token not populated for secret ${secret_name} in cluster ${member_cluster}" + exit 1 + fi # for 2 cluster tests central cluster is the first member, so we cannot add this as it will result in duplicate key and error in create secret if [[ "${member_cluster}" != "${CENTRAL_CLUSTER}" ]]; then configuration_params+=(