From 5b8272b10448e80b240867808992eefe95810869 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 17 Sep 2025 07:21:52 -0500 Subject: [PATCH] PYTHON-5472 Remove driver tests for Atlas Data Lake --- .evergreen/generated_configs/variants.yml | 11 --- .evergreen/resync-specs.sh | 3 - .evergreen/scripts/generate_config.py | 8 -- .evergreen/scripts/resync-all-specs.py | 1 - .evergreen/scripts/setup_tests.py | 12 +-- .evergreen/scripts/teardown_tests.py | 4 - .evergreen/scripts/utils.py | 2 - CONTRIBUTING.md | 7 -- pyproject.toml | 1 - test/__init__.py | 34 ------- test/asynchronous/__init__.py | 34 ------- test/asynchronous/test_data_lake.py | 107 ---------------------- test/test_data_lake.py | 107 ---------------------- tools/synchro.py | 1 - 14 files changed, 1 insertion(+), 331 deletions(-) delete mode 100644 test/asynchronous/test_data_lake.py delete mode 100644 test/test_data_lake.py diff --git a/.evergreen/generated_configs/variants.yml b/.evergreen/generated_configs/variants.yml index 33b8e0ba02..5e769173f9 100644 --- a/.evergreen/generated_configs/variants.yml +++ b/.evergreen/generated_configs/variants.yml @@ -79,17 +79,6 @@ buildvariants: TEST_NAME: atlas_connect tags: [pr] - # Atlas data lake tests - - name: atlas-data-lake-ubuntu-22 - tasks: - - name: .test-no-orchestration - display_name: Atlas Data Lake Ubuntu-22 - run_on: - - ubuntu2204-small - expansions: - TEST_NAME: data_lake - tags: [pr] - # Aws auth tests - name: auth-aws-ubuntu-20 tasks: diff --git a/.evergreen/resync-specs.sh b/.evergreen/resync-specs.sh index 765af2a562..d2bd89c781 100755 --- a/.evergreen/resync-specs.sh +++ b/.evergreen/resync-specs.sh @@ -76,9 +76,6 @@ do auth) cpjson auth/tests/ auth ;; - atlas-data-lake-testing|data_lake) - cpjson atlas-data-lake-testing/tests/ data_lake - ;; bson-binary-vector|bson_binary_vector) cpjson bson-binary-vector/tests/ bson_binary_vector ;; diff --git a/.evergreen/scripts/generate_config.py b/.evergreen/scripts/generate_config.py index f76c0d1e04..a04a64d30a 100644 --- a/.evergreen/scripts/generate_config.py +++ b/.evergreen/scripts/generate_config.py @@ -322,14 +322,6 @@ def create_no_c_ext_variants(): return [create_variant(tasks, display_name, host=host)] -def create_atlas_data_lake_variants(): - host = HOSTS["ubuntu22"] - tasks = [".test-no-orchestration"] - expansions = dict(TEST_NAME="data_lake") - display_name = get_variant_name("Atlas Data Lake", host) - return [create_variant(tasks, display_name, tags=["pr"], host=host, expansions=expansions)] - - def create_mod_wsgi_variants(): host = HOSTS["ubuntu22"] tasks = [".mod_wsgi"] diff --git a/.evergreen/scripts/resync-all-specs.py b/.evergreen/scripts/resync-all-specs.py index 0817e2fc3b..778a38af88 100644 --- a/.evergreen/scripts/resync-all-specs.py +++ b/.evergreen/scripts/resync-all-specs.py @@ -56,7 +56,6 @@ def check_new_spec_directories(directory: pathlib.Path) -> list[str]: "client_side_operations_timeout": "csot", "mongodb_handshake": "handshake", "load_balancers": "load_balancer", - "atlas_data_lake_testing": "atlas", "connection_monitoring_and_pooling": "connection_monitoring", "command_logging_and_monitoring": "command_logging", "initial_dns_seedlist_discovery": "srv_seedlist", diff --git a/.evergreen/scripts/setup_tests.py b/.evergreen/scripts/setup_tests.py index 9f383d9425..2ddede127d 100644 --- a/.evergreen/scripts/setup_tests.py +++ b/.evergreen/scripts/setup_tests.py @@ -214,18 +214,8 @@ def handle_test_env() -> None: if key in os.environ: write_env(key, os.environ[key]) - if test_name == "data_lake": - # Stop any running mongo-orchestration which might be using the port. - run_command(f"bash {DRIVERS_TOOLS}/.evergreen/stop-orchestration.sh") - run_command(f"bash {DRIVERS_TOOLS}/.evergreen/atlas_data_lake/setup.sh") - AUTH = "auth" - if AUTH != "noauth": - if test_name == "data_lake": - config = read_env(f"{DRIVERS_TOOLS}/.evergreen/atlas_data_lake/secrets-export.sh") - DB_USER = config["ADL_USERNAME"] - DB_PASSWORD = config["ADL_PASSWORD"] - elif test_name == "auth_oidc": + if test_name == "auth_oidc": DB_USER = config["OIDC_ADMIN_USER"] DB_PASSWORD = config["OIDC_ADMIN_PWD"] elif test_name == "search_index": diff --git a/.evergreen/scripts/teardown_tests.py b/.evergreen/scripts/teardown_tests.py index d89f513f12..7da0b60815 100644 --- a/.evergreen/scripts/teardown_tests.py +++ b/.evergreen/scripts/teardown_tests.py @@ -57,10 +57,6 @@ teardown_mod_wsgi() -# Tear down data_lake if applicable. -elif TEST_NAME == "data_lake": - run_command(f"{DRIVERS_TOOLS}/.evergreen/atlas_data_lake/teardown.sh") - # Tear down coverage if applicable. if os.environ.get("COVERAGE"): shutil.rmtree(".pytest_cache", ignore_errors=True) diff --git a/.evergreen/scripts/utils.py b/.evergreen/scripts/utils.py index 323ec2c567..8d1b466b6c 100644 --- a/.evergreen/scripts/utils.py +++ b/.evergreen/scripts/utils.py @@ -33,7 +33,6 @@ class Distro: "atlas_connect": "atlas_connect", "auth_aws": "auth_aws", "auth_oidc": "auth_oidc", - "data_lake": "data_lake", "default": "", "default_async": "default_async", "default_sync": "default", @@ -57,7 +56,6 @@ class Distro: "auth_oidc", "atlas_connect", "aws_lambda", - "data_lake", "mockupdb", "ocsp", ] diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ed52727765..16907d4285 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -355,13 +355,6 @@ Note: these tests can only be run from an Evergreen Linux host that has the Pyth The `mode` can be `standalone` or `embedded`. For the `replica_set` version of the tests, use `TOPOLOGY=replica_set just run-server`. -### Atlas Data Lake tests. - -You must have `docker` or `podman` installed locally. - -- Run `just setup-tests data_lake`. -- Run `just run-tests`. - ### OCSP tests - Export the orchestration file, e.g. `export ORCHESTRATION_FILE=rsa-basic-tls-ocsp-disableStapling.json`. diff --git a/pyproject.toml b/pyproject.toml index fe277d8ed0..ad256a6c6f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -134,7 +134,6 @@ markers = [ "auth: tests that rely on authentication", "ocsp: tests that rely on ocsp", "atlas_connect: tests that rely on an atlas connection", - "data_lake: tests that rely on atlas data lake", "perf: benchmark tests", "search_index: search index helper tests", "kms: client-side field-level encryption tests using kms", diff --git a/test/__init__.py b/test/__init__.py index d583a72f0f..bef672b5f7 100644 --- a/test/__init__.py +++ b/test/__init__.py @@ -121,7 +121,6 @@ def __init__(self): self.sessions_enabled = False self.client = None # type: ignore self.conn_lock = threading.Lock() - self.is_data_lake = False self.load_balancer = TEST_LOADBALANCER self._fips_enabled = None if self.load_balancer: @@ -199,16 +198,6 @@ def _init_client(self): self.mongoses = [] self.connection_attempts = [] self.client = self._connect(host, port) - if self.client is not None: - # Return early when connected to dataLake as mongohoused does not - # support the getCmdLineOpts command and is tested without TLS. - if os.environ.get("TEST_DATA_LAKE"): - self.is_data_lake = True - self.auth_enabled = True - self.client.close() - self.client = self._connect(host, port, username=db_user, password=db_pwd) - self.connected = True - return if HAVE_SSL and not self.client: # Is MongoDB configured for SSL? @@ -501,14 +490,6 @@ def require_connection(self, func): func=func, ) - def require_data_lake(self, func): - """Run a test only if we are connected to Atlas Data Lake.""" - return self._require( - lambda: self.is_data_lake, - "Not connected to Atlas Data Lake on self.pair", - func=func, - ) - def require_version_min(self, *ver): """Run a test only if the server version is at least ``version``.""" other_version = Version(*ver) @@ -1230,21 +1211,6 @@ def teardown(): garbage.append(f" gc.get_referrers: {gc.get_referrers(g)!r}") if garbage: raise AssertionError("\n".join(garbage)) - c = client_context.client - if c: - if not client_context.is_data_lake: - try: - c.drop_database("pymongo-pooling-tests") - c.drop_database("pymongo_test") - c.drop_database("pymongo_test1") - c.drop_database("pymongo_test2") - c.drop_database("pymongo_test_mike") - c.drop_database("pymongo_test_bernie") - except AutoReconnect: - # PYTHON-4982 - if sys.implementation.name.lower() != "pypy": - raise - c.close() print_running_clients() diff --git a/test/asynchronous/__init__.py b/test/asynchronous/__init__.py index 8ab7ff7219..5a65311808 100644 --- a/test/asynchronous/__init__.py +++ b/test/asynchronous/__init__.py @@ -121,7 +121,6 @@ def __init__(self): self.sessions_enabled = False self.client = None # type: ignore self.conn_lock = threading.Lock() - self.is_data_lake = False self.load_balancer = TEST_LOADBALANCER self._fips_enabled = None if self.load_balancer: @@ -199,16 +198,6 @@ async def _init_client(self): self.mongoses = [] self.connection_attempts = [] self.client = await self._connect(host, port) - if self.client is not None: - # Return early when connected to dataLake as mongohoused does not - # support the getCmdLineOpts command and is tested without TLS. - if os.environ.get("TEST_DATA_LAKE"): - self.is_data_lake = True - self.auth_enabled = True - await self.client.close() - self.client = await self._connect(host, port, username=db_user, password=db_pwd) - self.connected = True - return if HAVE_SSL and not self.client: # Is MongoDB configured for SSL? @@ -501,14 +490,6 @@ def require_connection(self, func): func=func, ) - def require_data_lake(self, func): - """Run a test only if we are connected to Atlas Data Lake.""" - return self._require( - lambda: self.is_data_lake, - "Not connected to Atlas Data Lake on self.pair", - func=func, - ) - def require_version_min(self, *ver): """Run a test only if the server version is at least ``version``.""" other_version = Version(*ver) @@ -1246,21 +1227,6 @@ async def async_teardown(): garbage.append(f" gc.get_referrers: {gc.get_referrers(g)!r}") if garbage: raise AssertionError("\n".join(garbage)) - c = async_client_context.client - if c: - if not async_client_context.is_data_lake: - try: - await c.drop_database("pymongo-pooling-tests") - await c.drop_database("pymongo_test") - await c.drop_database("pymongo_test1") - await c.drop_database("pymongo_test2") - await c.drop_database("pymongo_test_mike") - await c.drop_database("pymongo_test_bernie") - except AutoReconnect: - # PYTHON-4982 - if sys.implementation.name.lower() != "pypy": - raise - await c.close() print_running_clients() diff --git a/test/asynchronous/test_data_lake.py b/test/asynchronous/test_data_lake.py deleted file mode 100644 index 689bf38534..0000000000 --- a/test/asynchronous/test_data_lake.py +++ /dev/null @@ -1,107 +0,0 @@ -# Copyright 2020-present MongoDB, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Test Atlas Data Lake.""" -from __future__ import annotations - -import os -import sys -from pathlib import Path - -import pytest - -sys.path[0:0] = [""] - -from test.asynchronous import AsyncIntegrationTest, AsyncUnitTest, async_client_context, unittest -from test.asynchronous.unified_format import generate_test_classes -from test.utils_shared import ( - OvertCommandListener, -) - -from pymongo.asynchronous.helpers import anext - -_IS_SYNC = False - -pytestmark = pytest.mark.data_lake - - -class TestDataLakeMustConnect(AsyncUnitTest): - async def test_connected_to_data_lake(self): - self.assertTrue( - async_client_context.is_data_lake and async_client_context.connected, - "client context must be connected to data lake when DATA_LAKE is set. Failed attempts:\n{}".format( - async_client_context.connection_attempt_info() - ), - ) - - -class TestDataLakeProse(AsyncIntegrationTest): - # Default test database and collection names. - TEST_DB = "test" - TEST_COLLECTION = "driverdata" - - @async_client_context.require_data_lake - async def asyncSetUp(self): - await super().asyncSetUp() - - # Test killCursors - async def test_1(self): - listener = OvertCommandListener() - client = await self.async_rs_or_single_client(event_listeners=[listener]) - cursor = client[self.TEST_DB][self.TEST_COLLECTION].find({}, batch_size=2) - await anext(cursor) - - # find command assertions - find_cmd = listener.succeeded_events[-1] - self.assertEqual(find_cmd.command_name, "find") - cursor_id = find_cmd.reply["cursor"]["id"] - cursor_ns = find_cmd.reply["cursor"]["ns"] - - # killCursors command assertions - await cursor.close() - started = listener.started_events[-1] - self.assertEqual(started.command_name, "killCursors") - succeeded = listener.succeeded_events[-1] - self.assertEqual(succeeded.command_name, "killCursors") - - self.assertIn(cursor_id, started.command["cursors"]) - target_ns = ".".join([started.command["$db"], started.command["killCursors"]]) - self.assertEqual(cursor_ns, target_ns) - - self.assertIn(cursor_id, succeeded.reply["cursorsKilled"]) - - # Test no auth - async def test_2(self): - client = await self.async_rs_client_noauth() - await client.admin.command("ping") - - # Test with auth - async def test_3(self): - for mechanism in ["SCRAM-SHA-1", "SCRAM-SHA-256"]: - client = await self.async_rs_or_single_client(authMechanism=mechanism) - await client[self.TEST_DB][self.TEST_COLLECTION].find_one() - - -# Location of JSON test specifications. -if _IS_SYNC: - TEST_PATH = Path(__file__).parent / "data_lake/unified" -else: - TEST_PATH = Path(__file__).parent.parent / "data_lake/unified" - -# Generate unified tests. -globals().update(generate_test_classes(TEST_PATH, module=__name__)) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/test_data_lake.py b/test/test_data_lake.py deleted file mode 100644 index d6d2007007..0000000000 --- a/test/test_data_lake.py +++ /dev/null @@ -1,107 +0,0 @@ -# Copyright 2020-present MongoDB, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Test Atlas Data Lake.""" -from __future__ import annotations - -import os -import sys -from pathlib import Path - -import pytest - -sys.path[0:0] = [""] - -from test import IntegrationTest, UnitTest, client_context, unittest -from test.unified_format import generate_test_classes -from test.utils_shared import ( - OvertCommandListener, -) - -from pymongo.synchronous.helpers import next - -_IS_SYNC = True - -pytestmark = pytest.mark.data_lake - - -class TestDataLakeMustConnect(UnitTest): - def test_connected_to_data_lake(self): - self.assertTrue( - client_context.is_data_lake and client_context.connected, - "client context must be connected to data lake when DATA_LAKE is set. Failed attempts:\n{}".format( - client_context.connection_attempt_info() - ), - ) - - -class TestDataLakeProse(IntegrationTest): - # Default test database and collection names. - TEST_DB = "test" - TEST_COLLECTION = "driverdata" - - @client_context.require_data_lake - def setUp(self): - super().setUp() - - # Test killCursors - def test_1(self): - listener = OvertCommandListener() - client = self.rs_or_single_client(event_listeners=[listener]) - cursor = client[self.TEST_DB][self.TEST_COLLECTION].find({}, batch_size=2) - next(cursor) - - # find command assertions - find_cmd = listener.succeeded_events[-1] - self.assertEqual(find_cmd.command_name, "find") - cursor_id = find_cmd.reply["cursor"]["id"] - cursor_ns = find_cmd.reply["cursor"]["ns"] - - # killCursors command assertions - cursor.close() - started = listener.started_events[-1] - self.assertEqual(started.command_name, "killCursors") - succeeded = listener.succeeded_events[-1] - self.assertEqual(succeeded.command_name, "killCursors") - - self.assertIn(cursor_id, started.command["cursors"]) - target_ns = ".".join([started.command["$db"], started.command["killCursors"]]) - self.assertEqual(cursor_ns, target_ns) - - self.assertIn(cursor_id, succeeded.reply["cursorsKilled"]) - - # Test no auth - def test_2(self): - client = self.rs_client_noauth() - client.admin.command("ping") - - # Test with auth - def test_3(self): - for mechanism in ["SCRAM-SHA-1", "SCRAM-SHA-256"]: - client = self.rs_or_single_client(authMechanism=mechanism) - client[self.TEST_DB][self.TEST_COLLECTION].find_one() - - -# Location of JSON test specifications. -if _IS_SYNC: - TEST_PATH = Path(__file__).parent / "data_lake/unified" -else: - TEST_PATH = Path(__file__).parent.parent / "data_lake/unified" - -# Generate unified tests. -globals().update(generate_test_classes(TEST_PATH, module=__name__)) - - -if __name__ == "__main__": - unittest.main() diff --git a/tools/synchro.py b/tools/synchro.py index e502f96281..a4190529c4 100644 --- a/tools/synchro.py +++ b/tools/synchro.py @@ -230,7 +230,6 @@ def async_only_test(f: str) -> bool: "test_cursor.py", "test_custom_types.py", "test_database.py", - "test_data_lake.py", "test_discovery_and_monitoring.py", "test_dns.py", "test_encryption.py",