diff --git a/source/load-balancers/tests/cursors.json b/source/load-balancers/tests/cursors.json index 43e4fbb4f6..6eddc0ebc8 100644 --- a/source/load-balancers/tests/cursors.json +++ b/source/load-balancers/tests/cursors.json @@ -902,6 +902,11 @@ }, { "description": "listCollections pins the cursor to a connection", + "runOnRequirements": [ + { + "serverless": "forbid" + } + ], "operations": [ { "name": "listCollections", @@ -1151,6 +1156,11 @@ }, { "description": "change streams pin to a connection", + "runOnRequirements": [ + { + "serverless": "forbid" + } + ], "operations": [ { "name": "createChangeStream", diff --git a/source/load-balancers/tests/cursors.yml b/source/load-balancers/tests/cursors.yml index 0797d0c93a..b99f4a26bd 100644 --- a/source/load-balancers/tests/cursors.yml +++ b/source/load-balancers/tests/cursors.yml @@ -357,6 +357,8 @@ tests: - connectionCheckedInEvent: {} - description: listCollections pins the cursor to a connection + runOnRequirements: + - serverless: forbid # CLOUDP-98562 listCollections batchSize is ignored on serverless. operations: - name: listCollections object: *database0 @@ -464,6 +466,8 @@ tests: - connectionCheckedInEvent: {} - description: change streams pin to a connection + runOnRequirements: + - serverless: forbid # Serverless does not support change streams. operations: - name: createChangeStream object: *collection0 diff --git a/source/load-balancers/tests/sdam-error-handling.json b/source/load-balancers/tests/sdam-error-handling.json index 63aabc04db..c0f114cdfb 100644 --- a/source/load-balancers/tests/sdam-error-handling.json +++ b/source/load-balancers/tests/sdam-error-handling.json @@ -127,6 +127,11 @@ "tests": [ { "description": "only connections for a specific serviceId are closed when pools are cleared", + "runOnRequirements": [ + { + "serverless": "forbid" + } + ], "operations": [ { "name": "createFindCursor", diff --git a/source/load-balancers/tests/sdam-error-handling.yml b/source/load-balancers/tests/sdam-error-handling.yml index 9ed1b226fb..0e6c8993af 100644 --- a/source/load-balancers/tests/sdam-error-handling.yml +++ b/source/load-balancers/tests/sdam-error-handling.yml @@ -65,6 +65,10 @@ initialData: tests: - description: only connections for a specific serviceId are closed when pools are cleared + runOnRequirements: + # This test assumes that two sequential connections receive different serviceIDs. + # Sequential connections to a serverless instance may receive the same serviceID. + - serverless: forbid operations: # Create two cursors to force two connections. - name: createFindCursor diff --git a/source/load-balancers/tests/transactions.json b/source/load-balancers/tests/transactions.json index add2453848..8cf24f4ca4 100644 --- a/source/load-balancers/tests/transactions.json +++ b/source/load-balancers/tests/transactions.json @@ -607,6 +607,11 @@ }, { "description": "pinned connection is released after a transient non-network CRUD error", + "runOnRequirements": [ + { + "serverless": "forbid" + } + ], "operations": [ { "name": "failPoint", @@ -715,6 +720,11 @@ }, { "description": "pinned connection is released after a transient network CRUD error", + "runOnRequirements": [ + { + "serverless": "forbid" + } + ], "operations": [ { "name": "failPoint", @@ -831,6 +841,11 @@ }, { "description": "pinned connection is released after a transient non-network commit error", + "runOnRequirements": [ + { + "serverless": "forbid" + } + ], "operations": [ { "name": "failPoint", diff --git a/source/load-balancers/tests/transactions.yml b/source/load-balancers/tests/transactions.yml index 90f9d4a09e..29cbbee723 100644 --- a/source/load-balancers/tests/transactions.yml +++ b/source/load-balancers/tests/transactions.yml @@ -240,6 +240,8 @@ tests: - connectionCheckedInEvent: {} - description: pinned connection is released after a transient non-network CRUD error + runOnRequirements: + - serverless: forbid # (CLOUDP-88216) Serverless does not append error labels to errors triggered by failpoints. operations: - name: failPoint object: testRunner @@ -280,6 +282,8 @@ tests: - connectionCheckedInEvent: {} - description: pinned connection is released after a transient network CRUD error + runOnRequirements: + - serverless: forbid # (CLOUDP-88216) Serverless does not append error labels to errors triggered by failpoints. operations: - name: failPoint object: testRunner @@ -323,6 +327,8 @@ tests: - connectionCheckedInEvent: {} - description: pinned connection is released after a transient non-network commit error + runOnRequirements: + - serverless: forbid # (CLOUDP-88216) Serverless does not append error labels to errors triggered by failpoints. operations: - name: failPoint object: testRunner diff --git a/source/retryable-reads/tests/README.rst b/source/retryable-reads/tests/README.rst index 68fbb192ec..0661bcaa19 100644 --- a/source/retryable-reads/tests/README.rst +++ b/source/retryable-reads/tests/README.rst @@ -109,10 +109,18 @@ Each YAML file has the following keys: - ``clientOptions``: Optional, parameters to pass to MongoClient(). - - ``useMultipleMongoses`` (optional): If ``true``, the MongoClient for this - test should be initialized with multiple mongos seed addresses. If ``false`` - or omitted, only a single mongos address should be specified. This field has - no effect for non-sharded topologies. + - ``useMultipleMongoses`` (optional): If ``true``, and the topology type is + ``Sharded``, the MongoClient for this test should be initialized with multiple + mongos seed addresses. If ``false`` or omitted, only a single mongos address + should be specified. + + If ``true``, and the topology type is ``LoadBalanced``, the MongoClient for + this test should be initialized with the URI of the load balancer fronting + multiple servers. If ``false`` or omitted, the MongoClient for this test + should be initialized with the URI of the load balancer fronting a single + server. + + ``useMultipleMongoses`` only affects ``Sharded`` and ``LoadBalanced`` topologies. - ``skipReason``: Optional, string describing why this test should be skipped. @@ -221,6 +229,8 @@ This test requires MongoDB 4.2.9+ for ``blockConnection`` support in the failpoi Changelog ========= +:2021-08-27: Clarify behavior of ``useMultipleMongoses`` for ``LoadBalanced`` topologies. + :2019-03-19: Add top-level ``runOn`` field to denote server version and/or topology requirements requirements for the test file. Removes the ``minServerVersion`` and ``topology`` top-level fields, which are diff --git a/source/retryable-writes/tests/README.rst b/source/retryable-writes/tests/README.rst index 3fa7713497..7cc234d51c 100644 --- a/source/retryable-writes/tests/README.rst +++ b/source/retryable-writes/tests/README.rst @@ -169,6 +169,18 @@ Each YAML file has the following keys: the default is all topologies (i.e. ``["single", "replicaset", "sharded", "load-balanced"]``). + - ``serverless``: Optional string. Whether or not the test should be run on + serverless instances imitating sharded clusters. Valid values are "require", + "forbid", and "allow". If "require", the test MUST only be run on serverless + instances. If "forbid", the test MUST NOT be run on serverless instances. If + omitted or "allow", this option has no effect. + + The test runner MUST be informed whether or not serverless is being used in + order to determine if this requirement is met (e.g. through an environment + variable or configuration option). Since the serverless proxy imitates a + mongos, the runner is not capable of determining this by issuing a server + command such as ``buildInfo`` or ``hello``. + - ``data``: The data that should exist in the collection under test before each test run. @@ -179,10 +191,18 @@ Each YAML file has the following keys: - ``clientOptions``: Parameters to pass to MongoClient(). - - ``useMultipleMongoses`` (optional): If ``true``, the MongoClient for this - test should be initialized with multiple mongos seed addresses. If ``false`` - or omitted, only a single mongos address should be specified. This field has - no effect for non-sharded topologies. + - ``useMultipleMongoses`` (optional): If ``true``, and the topology type is + ``Sharded``, the MongoClient for this test should be initialized with multiple + mongos seed addresses. If ``false`` or omitted, only a single mongos address + should be specified. + + If ``true``, and the topology type is ``LoadBalanced``, the MongoClient for + this test should be initialized with the URI of the load balancer fronting + multiple servers. If ``false`` or omitted, the MongoClient for this test + should be initialized with the URI of the load balancer fronting a single + server. + + ``useMultipleMongoses`` only affects ``Sharded`` and ``LoadBalanced`` topologies. - ``failPoint`` (optional): The ``configureFailPoint`` command document to run to configure a fail point on the primary server. Drivers must ensure that @@ -374,6 +394,11 @@ and sharded clusters. Changelog ========= + + +:2021-08-27: Add ``serverless`` to ``runOn``. Clarify behavior of + ``useMultipleMongoses`` for ``LoadBalanced`` topologies. + :2021-04-23: Add ``load-balanced`` to test topology requirements. :2021-03-24: Add prose test verifying ``PoolClearedErrors`` are retried. diff --git a/source/serverless-testing/README.rst b/source/serverless-testing/README.rst new file mode 100644 index 0000000000..6a6b06399a --- /dev/null +++ b/source/serverless-testing/README.rst @@ -0,0 +1,138 @@ +====================== +Atlas Serverless Tests +====================== + +.. contents:: + +---- + +Introduction +============ + +This file describes a subset of existing tests that drivers MUST use to assert +compatibility with Atlas Serverless. + +Serverless Configuration +======================== + +These tests MUST be run against a live Atlas Serverless instance. A new instance +MUST be created each time the test suite is run, and that instance MUST be used +for all of the tests required by this specification. Once the tests are +finished, the instance MUST be deleted regardless of the outcome of the tests. +The `serverless directory in the drivers-evergreen-tools repository`_ contains +scripts for creating and deleting Atlas Serverless instances, and the +``config.yml`` contains an example Evergreen configuration that uses them to run +the tests. It can take up to 15 minutes or so to provision a new Atlas +Serverless instance, so it is recommended to create one manually via the scripts +in drivers-evergreen-tools that can be reused for the initial implementation of +the tests before moving to Evergreen patches. + +Drivers MUST use the `create-instance.sh`_ script in ``drivers-evergreen-tools`` +to create a new Atlas Serverless instance for Evergreen tasks. The script writes +two URIs to a YAML expansions file: + +- ``MULTI_ATLASPROXY_SERVERLESS_URI`` A URI to the load balancer fronting + multiple Atlas Proxy processes. + +- ``SINGLE_ATLASPROXY_SERVERLESS_URI`` A URI to one of the Atlas Proxy + processes. + +The URIs can be read by drivers via the ``expansions.update`` Evergreen command. +This will store the URIs into the ``SINGLE_ATLASPROXY_SERVERLESS_URI`` and +``MULTI_ATLASPROXY_SERVERLESS_URI`` Evergreen expansions. + +.. _serverless directory in the drivers-evergreen-tools repository: https://github.com/mongodb-labs/drivers-evergreen-tools/tree/1ca6209825b6ed07ce90e24cda659143443709c8/.evergreen/serverless + +All tests MUST be run with wire protocol compression and authentication +enabled. + +Test Runner Configuration +========================= + +Drivers MUST add the username and password to both +``SINGLE_ATLASPROXY_SERVERLESS_URI`` and ``MULTI_ATLASPROXY_SERVERLESS_URI`` to +ensure that test clients can connect to the cluster. Drivers MUST use the final +URI stored in ``SINGLE_ATLASPROXY_SERVERLESS_URI`` to configure internal clients +for test runners (e.g. the internal MongoClient described by the `Unified Test +Format spec <../../unified-test-format/unified-test-format.rst>`__). + +The ``SINGLE_ATLASPROXY_SERVERLESS_URI`` and ``MULTI_ATLASPROXY_SERVERLESS_URI`` +URIs both contain ``loadBalanced=true`` as a URI option. + +.. _Load Balancer testing README: https://github.com/mongodb/specifications/blob/5fad2773960d63d46b7738dde8d7b1aca0ceaac9/source/load-balancers/tests/README.rst#load-balancer-configuration + +``SINGLE_ATLASPROXY_SERVERLESS_URI`` MUST be used in test runners where a load +balancer fronting a single server is required. It is analogous to the +``SINGLE_LB_MONGOS_URI`` in the `Load Balancer testing README`_. + +``MULTI_ATLASPROXY_SERVERLESS_URI`` MUST be used in test runners where a load +balancer fronting multiple servers is required. It is analogous to the +``MULTI_LB_MONGOS_URI`` in the `Load Balancer testing README`_. + +Required Variables +~~~~~~~~~~~~~~~~~~ + +Managing the Atlas Serverless instances and connecting to them requires a few +variables to be specified. The variables marked "private" are confidential and +MUST be specified as private Evergreen variables or used only in private +Evergreen projects. If using a public Evergreen project, xtrace MUST be disabled +when using these variables to help prevent accidental leaks. + +- ``${SERVERLESS_DRIVERS_GROUP}``: Contains the ID of the Atlas group dedicated + to drivers testing of Atlas Serverless. + +- ``${SERVERLESS_API_PUBLIC_KEY}``: The public key required to use the Atlas API + for provisioning of Atlas Serverless instances. + +- ``${SERVERLESS_API_PRIVATE_KEY}``: (private) The private key required to use + the Atlas API for provisioning of Atlas Serverless instances. + +- ``${SERVERLESS_ATLAS_USER}``: (private) The SCRAM username used to + authenticate to any Atlas Serverless instance created in the drivers testing + Atlas group. + +- ``${SERVERLESS_ATLAS_PASSWORD}``: (private) The SCRAM password used to + authenticate to any Atlas Serverless instance created in the drivers testing + Atlas group. + +Existing Spec Tests +=================== + +Tests defined in the following specifications MUST be included in a driver's +Atlas Serverless testing suite, including prose tests: + +- CRUD, including the v1 and unified tests +- Retryable Reads +- Retryable Writes +- Versioned API +- Driver Sessions +- Transactions (excluding convenient API) + + - Note: the killAllSessions command is not supported on Serverless, so the + transactions tests may hang if an individual test leaves a transaction open + when it finishes (`CLOUDP-84298 `_) +- Load Balancer unified tests + +.. _create-instance.sh: https://github.com/mongodb-labs/drivers-evergreen-tools/blob/1ca6209825b6ed07ce90e24cda659143443709c8/.evergreen/serverless/create-instance.sh + +Note that the formats for the JSON/YAML tests of these specifications were +updated to include a new ``runOnRequirement`` specifically for Atlas Serverless +testing, so make sure to update the affected test runners to support this +requirement and then sync the tests. To ensure these requirements are enforced +properly, the runner MUST be informed that it is running against an Atlas +Serverless instance through some indicator (e.g. an environment variable). + +Other Tests +=========== + +Any other existing tests for cursor behavior that a driver may have implemented +independently of any spec requirements SHOULD also be included in the driver's +Atlas Serverless testing suite. Note that ChangeStreams are not supported by the +proxy, so tests for them cannot be included. + + +Changelog +========= + +:2021-08-25: Update tests for load balanced serverless instances. +