From cd9e82a9401ba70b033236924a67666dc177094e Mon Sep 17 00:00:00 2001 From: Nam Nguyen Date: Tue, 22 Apr 2025 16:38:24 +0200 Subject: [PATCH] cherry-pick fix flaky scram auth --- .../kubetester/kubetester.py | 52 +++++++++++++++++++ ...harded_cluster_x509_to_scram_transition.py | 20 ++++++- 2 files changed, 70 insertions(+), 2 deletions(-) diff --git a/docker/mongodb-enterprise-tests/kubetester/kubetester.py b/docker/mongodb-enterprise-tests/kubetester/kubetester.py index dd2e96dd0..27c161cec 100644 --- a/docker/mongodb-enterprise-tests/kubetester/kubetester.py +++ b/docker/mongodb-enterprise-tests/kubetester/kubetester.py @@ -27,6 +27,7 @@ from kubernetes.stream import stream from kubetester.crypto import wait_for_certs_to_be_issued from requests.auth import HTTPBasicAuth, HTTPDigestAuth +from tests import test_logger SSL_CA_CERT = "/var/run/secrets/kubernetes.io/serviceaccount/..data/ca.crt" EXTERNALLY_MANAGED_TAG = "EXTERNALLY_MANAGED_BY_KUBERNETES" @@ -46,6 +47,11 @@ "MongoDBMultiCluster": "mongodbmulticluster", } +from opentelemetry import trace + +TRACER = trace.get_tracer("evergreen-agent") +logger = test_logger.get_test_logger(__name__) + def running_locally(): return os.getenv("POD_NAME", "local") == "local" @@ -946,6 +952,16 @@ def get_automation_config(group_id=None, group_name=None): return response.json() + @staticmethod + def get_automation_status(group_id=None, group_name=None): + if group_id is None: + group_id = KubernetesTester.get_om_group_id(group_name=group_name) + + url = build_automation_status_endpoint(KubernetesTester.get_om_base_url(), group_id) + response = KubernetesTester.om_request("get", url) + + return response.json() + @staticmethod def get_monitoring_config(group_id=None): if group_id is None: @@ -1544,6 +1560,10 @@ def build_automation_config_endpoint(base_url, group_id): return "{}/api/public/v1.0/groups/{}/automationConfig".format(base_url, group_id) +def build_automation_status_endpoint(base_url, group_id): + return "{}/api/public/v1.0/groups/{}/automationStatus".format(base_url, group_id) + + def build_monitoring_config_endpoint(base_url, group_id): return "{}/api/public/v1.0/groups/{}/automationConfig/monitoringAgentConfig".format(base_url, group_id) @@ -1683,3 +1703,35 @@ def ensure_ent_version(mdb_version: str) -> str: if "-ent" not in mdb_version: return mdb_version + "-ent" return mdb_version + + +@TRACER.start_as_current_span("wait_processes_ready") +def wait_processes_ready(): + # Get current automation status + def processes_are_ready(): + auto_status = KubernetesTester.get_automation_status() + goal_version = auto_status.get("goalVersion") + + logger.info(f"Checking if all processes have reached goal version: {goal_version}") + processes_not_ready = [] + for process in auto_status.get("processes", []): + process_name = process.get("name", "unknown") + process_version = process.get("lastGoalVersionAchieved") + if process_version != goal_version: + logger.info(f"Process {process_name} at version {process_version}, expected {goal_version}") + processes_not_ready.append(process_name) + + all_processes_ready = len(processes_not_ready) == 0 + if all_processes_ready: + logger.info("All processes have reached the goal version") + else: + logger.info(f"{len(processes_not_ready)} processes have not yet reached the goal version") + + return all_processes_ready + + timeout = 600 # 5 minutes timeout + KubernetesTester.wait_until( + processes_are_ready, + timeout=timeout, + sleep_time=5, + ) diff --git a/docker/mongodb-enterprise-tests/tests/authentication/sharded_cluster_x509_to_scram_transition.py b/docker/mongodb-enterprise-tests/tests/authentication/sharded_cluster_x509_to_scram_transition.py index 682539339..822c61048 100644 --- a/docker/mongodb-enterprise-tests/tests/authentication/sharded_cluster_x509_to_scram_transition.py +++ b/docker/mongodb-enterprise-tests/tests/authentication/sharded_cluster_x509_to_scram_transition.py @@ -1,4 +1,7 @@ +import time + import pytest +from kubetester import kubetester from kubetester.automation_config_tester import AutomationConfigTester from kubetester.certs import ( ISSUER_CA_NAME, @@ -12,11 +15,13 @@ from kubetester.mongotester import ShardedClusterTester from kubetester.omtester import get_sc_cert_names from pytest import fixture +from tests import test_logger MDB_RESOURCE = "sharded-cluster-x509-to-scram-256" USER_NAME = "mms-user-1" PASSWORD_SECRET_NAME = "mms-user-1-password" USER_PASSWORD = "my-password" +logger = test_logger.get_test_logger(__name__) @fixture(scope="module") @@ -76,6 +81,8 @@ def test_x509_is_still_configured(): @pytest.mark.e2e_sharded_cluster_x509_to_scram_transition class TestShardedClusterDisableAuthentication(KubernetesTester): def test_disable_auth(self, sharded_cluster: MongoDB): + kubetester.wait_processes_ready() + sharded_cluster.assert_reaches_phase(Phase.Running, timeout=800) sharded_cluster.load() sharded_cluster["spec"]["security"]["authentication"]["enabled"] = False sharded_cluster.update() @@ -90,9 +97,18 @@ def test_ops_manager_state_updated_correctly(self): tester.assert_authentication_disabled() +from opentelemetry import trace + +TRACER = trace.get_tracer("evergreen-agent") + + @pytest.mark.e2e_sharded_cluster_x509_to_scram_transition class TestCanEnableScramSha256: - def test_can_enable_scram_sha_256(self, sharded_cluster: MongoDB): + @TRACER.start_as_current_span("test_can_enable_scram_sha_256") + def test_can_enable_scram_sha_256(self, sharded_cluster: MongoDB, ca_path: str): + kubetester.wait_processes_ready() + sharded_cluster.assert_reaches_phase(Phase.Running, timeout=800) + sharded_cluster.load() sharded_cluster["spec"]["security"]["authentication"]["enabled"] = True sharded_cluster["spec"]["security"]["authentication"]["modes"] = [ @@ -100,7 +116,7 @@ def test_can_enable_scram_sha_256(self, sharded_cluster: MongoDB): ] sharded_cluster["spec"]["security"]["authentication"]["agents"]["mode"] = "SCRAM" sharded_cluster.update() - sharded_cluster.assert_reaches_phase(Phase.Running, timeout=1200) + sharded_cluster.assert_reaches_phase(Phase.Running, timeout=800) def test_assert_connectivity(self, ca_path: str): ShardedClusterTester(MDB_RESOURCE, 1, ssl=True, ca_path=ca_path).assert_connectivity(attempts=25)