feat: Enable multiprocess (xdist) use of the redis fixture.

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "pytest-mock-resources"
-version = "2.1.8"
+version = "2.1.9"
 description = "A pytest plugin for easily instantiating reproducible mock resources."
 authors = [
     "Omar Khan <oakhan3@gmail.com>",

setup.cfg

@@ -5,7 +5,8 @@ match_dir = ^[^\.{]((?!igrations).)*
 [tool:pytest]
 doctest_optionflags = NORMALIZE_WHITESPACE IGNORE_EXCEPTION_DETAIL ELLIPSIS
 addopts = --ff --doctest-modules
-norecursedirs = .* build dist *.egg
+norecursedirs = .* build dist *.egg tests/examples
+pytester_example_dir = tests/examples
 markers =
     postgres
     redshift

src/pytest_mock_resources/fixture/database/generic.py

@@ -14,7 +14,7 @@ def __init__(self, drivername, host, port, database, username, password):
         self.drivername = drivername
         self.host = host
         self.port = port
-        self.database = database
+        self.database = str(database)
         self.username = username
         self.password = password
 
@@ -69,7 +69,7 @@ def as_redis_kwargs(self):
         return {
             "host": self.host,
             "port": self.port,
-            "db": self.database,
+            "db": int(self.database),
             "username": self.username,
             "password": self.password,
         }

src/pytest_mock_resources/fixture/database/redis.py

@@ -23,6 +23,19 @@ def create_redis_fixture(scope="function"):
     Any number of fixture functions can be created. Under the hood they will all share the same
     database server.
 
+    .. note::
+
+       If running tests in parallel, the implementation fans out to different redis "database"s,
+       up to a 16 (which is the default container fixed limite). This means you can only run
+       up to 16 simultaneous tests.
+
+       Additionally, any calls to `flushall` or any other cross-database calls **will** still
+       represent cross-test state.
+
+       Finally, the above notes are purely describing the current implementation, and should not
+       be assumed. In the future, the current database selection mechanism may change, or
+       databases may not be used altogether.
+
     Args:
         scope (str): The scope of the fixture can be specified by the user, defaults to "function".
 
@@ -31,15 +44,25 @@ def create_redis_fixture(scope="function"):
     """
 
     @pytest.fixture(scope=scope)
-    def _(_redis_container, pmr_redis_config):
-        db = redis.Redis(host=pmr_redis_config.host, port=pmr_redis_config.port)
-        db.flushall()
+    def _(request, _redis_container, pmr_redis_config):
+        database_number = 0
+        if hasattr(request.config, 'workerinput'):
+            worker_input = request.config.workerinput
+            worker_id = worker_input['workerid']  # For example "gw0".
+            database_number = int(worker_id[2:])
+
+        if database_number >= 16:
+            raise ValueError("The redis fixture currently only supports up to 16 parallel executions")
+
+        db = redis.Redis(host=pmr_redis_config.host, port=pmr_redis_config.port, db=database_number)
+        db.flushdb()
+
         assign_fixture_credentials(
             db,
             drivername="redis",
             host=pmr_redis_config.host,
             port=pmr_redis_config.port,
-            database=None,
+            database=database_number,
             username=None,
             password=None,
         )

tests/conftest.py

@@ -1,5 +1,7 @@
 import pytest
 
+pytest_plugins = "pytester"
+
 
 # See https://github.com/spulec/moto/issues/3292#issuecomment-770682026
 @pytest.fixture(autouse=True)

tests/examples/test_multiprocess_redis_database/conftest.py

@@ -0,0 +1,3 @@
+from pytest_mock_resources import create_redis_fixture
+
+redis = create_redis_fixture()

tests/examples/test_multiprocess_redis_database/test_split.py

@@ -0,0 +1,47 @@
+"""Produce a test example which would induce a redis database related race condition.
+
+The premise is that given a pytest invocation: `pytest -n 4 test_split.py`,
+multiple processes would start up simultaneously, and all connecting to the same
+redis database.
+
+The tests would then proceed to clobber one another's values, leading to flaky
+tests.
+
+A correct implementation would use some mechanism to avoid this inter-parallel-test
+key conflict problem.
+"""
+import random
+import time
+
+
+def test_node_one(redis, pytestconfig):
+    run_test(redis, pytestconfig)
+
+
+def test_node_two(redis, pytestconfig):
+    run_test(redis, pytestconfig)
+
+
+def test_node_three(redis, pytestconfig):
+    run_test(redis, pytestconfig)
+
+
+def test_node_four(redis, pytestconfig):
+    run_test(redis, pytestconfig)
+
+
+def run_test(redis, pytestconfig):
+    worker_id = int(pytestconfig.workerinput['workerid'][2:])
+    database = redis.connection_pool.get_connection('set').db
+    assert worker_id == database
+    print(worker_id, database)
+
+    redis.set("foo", "bar")
+    # XXX: Ideally we would sleep random times to ensure we're hitting the problem,
+    # XXX: however until the plugin is overall more process-safe, it's too flaky.
+    # time.sleep(random.randrange(1, 10) / 10)
+    value = redis.get("foo")
+
+    assert value == b"bar"
+
+    redis.flushdb()

tests/test_examples.py

@@ -0,0 +1,11 @@
+import pytest
+
+
+@pytest.mark.redis
+def test_multiprocess_redis_database(pytester):
+    pytester.copy_example()
+
+    # The `-n 2` are here is tightly coupled with the implementation of `test_split.py`.
+    args = ["-vv", "-n", "4", "test_split.py"]
+    result = pytester.inline_run(*args)
+    result.assertoutcome(passed=4, skipped=0, failed=0)