test: Run Pytest tests in parallel (meltano#6710)

* Run Pytest tests in parallel

Closes meltano#6157

* Install `pytest-xdist` in Nox env

* Use `pytest-cov` to get coverage data with `pytest-xdist`

* Set coverage results file based on test matrix ID

* Collect coverage data from all Python versions

.github/workflows/test.yml

@@ -132,26 +132,24 @@ jobs:
 
     - name: Run pytest
       env:
+        COLUMNS: 160
         PYTEST_BACKEND: ${{ matrix.backend-db }}
-
+        COVERAGE_FILE: .coverage.${{ matrix.id }}
         # Postgres backend
         POSTGRES_HOST_AUTH_METHOD: trust
         POSTGRES_ADDRESS: localhost
         POSTGRES_PORT: 5432
         POSTGRES_USER: postgres
         POSTGRES_PASSWORD: postgres
         POSTGRES_DB: pytest_warehouse
-
         # MSSQL backend
         MSSQL_ADDRESS: localhost
         MSSQL_PORT: 1433
         MSSQL_USER: sa
         MSSQL_PASSWORD: Meltan0admin
         MSSQL_DB: pytest_warehouse
-
       shell: bash
       run: |
-        export COLUMNS=160
         nox -rs tests --python=${{ matrix.python-version }} -- -m "${{ env.PYTEST_MARKERS }}" 2>&1 | \
           tee >( \
             tail | sed -r "s/[[:cntrl:]]\[([0-9]{1,3};)*[0-9]{1,3}m//g" | grep -E '^=+ [0-9].+ =+$' | \
@@ -177,11 +175,11 @@ jobs:
         $( pattern='in ([0-9]+)\.[0-9]+s' && [[ ${{ env.pytest_results }} =~ $pattern ]] && echo "${BASH_REMATCH[1]}" )s |"
 
     - name: Upload coverage data
-      if: always() && (matrix.python-version == '3.9')
+      if: always()
       uses: actions/upload-artifact@v3.1.0
       with:
         name: coverage-data
-        path: ".coverage.*"
+        path: ".coverage.${{ matrix.id }}"
 
   summary:
     runs-on: ubuntu-latest

noxfile.py

@@ -40,23 +40,20 @@ def tests(session: Session) -> None:
         session.install(".")
 
     session.install(
-        "coverage[toml]",
         "freezegun",
         "mock",
         "pytest",
         "pytest-asyncio",
+        "pytest-cov",
         "pytest-docker",
         "pytest-order",
         "pytest-randomly",
+        "pytest-xdist",
         "requests-mock",
     )
 
     try:
         session.run(
-            "coverage",
-            "run",
-            "--parallel",
-            "-m",
             "pytest",
             f"--randomly-seed={randint(0, 2**32-1)}",  # noqa: S311, WPS432
             *session.posargs,

pyproject.toml

@@ -91,8 +91,10 @@ pytest-cov = "^3.0.0"
 pytest-docker = "^1.0"
 pytest-order = "^1.0"
 pytest-randomly = "^3.12"
+pytest-xdist = "^2.5"
 pyupgrade = "^2.29.1"
 requests-mock = "^1.6.0"
+setproctitle = "^1.3" # Used by pytest-xdist to aid with handling resource intensive processes.
 tox = "^3.24.4"
 types-requests = "^2.28.9"
 wemake-python-styleguide = "^0.16.1"

pytest.ini

@@ -1,5 +1,5 @@
 [pytest]
-addopts = --doctest-modules --order-scope=module -ra
+addopts = --cov=meltano --cov=tests --doctest-modules --order-scope=module -ra -n auto --dist=loadfile
 testpaths = tests
 
 markers =

tests/fixtures/db/__init__.py

@@ -3,7 +3,6 @@
 import logging
 import warnings
 from contextlib import closing
-from time import sleep
 from typing import Generator
 
 import pytest

tests/fixtures/db/mssql.py

@@ -64,12 +64,12 @@ def create_connection_url(
 
 
 @pytest.fixture(scope="session")
-def engine_uri():
+def engine_uri(worker_id: str):
     host = os.getenv("MSSQL_ADDRESS")
     port = os.getenv("MSSQL_PORT", 1433)
     user = os.getenv("MSSQL_USER")
     password = os.getenv("MSSQL_PASSWORD")
-    database = os.getenv("MSSQL_DB", "pytest_meltano")
+    database = f"{os.getenv('MSSQL_DB', 'pytest_meltano')}_{worker_id}"
 
     # Recreate the database using the master database
     master_engine_uri = create_connection_url(host, port, user, password, "master")

tests/fixtures/db/postgresql.py

@@ -19,12 +19,12 @@ def recreate_database(engine, db_name):
 
 
 @pytest.fixture(scope="session")
-def engine_uri():
+def engine_uri(worker_id: str):
     host = os.getenv("POSTGRES_ADDRESS")
     port = os.getenv("POSTGRES_PORT", 5432)
     user = os.getenv("POSTGRES_USER")
     password = os.getenv("POSTGRES_PASSWORD")
-    database = os.getenv("POSTGRES_DB", "pytest_meltano")
+    database = f"{os.getenv('POSTGRES_DB', 'pytest_meltano')}_{worker_id}"
 
     # create the database
     engine_uri = f"postgresql://{user}:{password}@{host}:{port}/postgres"

tests/fixtures/db/sqlite.py

@@ -4,8 +4,9 @@
 
 
 @pytest.fixture(scope="session")
-def engine_uri(tmp_path_factory) -> str:
+def engine_uri(tmp_path_factory, worker_id: str) -> str:
     database_path = (
-        tmp_path_factory.mktemp("fixture_db_sqlite_engine_uri") / "pytest_meltano.db"
+        tmp_path_factory.mktemp("fixture_db_sqlite_engine_uri")
+        / f"pytest_meltano_{worker_id}.db"
     )
     return f"sqlite:///{database_path.as_posix()}"

tests/meltano/core/tracking/test_tracker.py

@@ -173,7 +173,7 @@ def test_analytics_json_is_created(self, project: Project):
             f'["{str(uuid.uuid4())}","{str(uuid.uuid4())}", true]',
             f'client_id":"{str(uuid.uuid4())}","project_id":"{str(uuid.uuid4())}","send_anonymous_usage_stats":true}}',  # noqa: E501
         ],
-        ids=lambda param: hash_sha256(param)[:8],
+        ids=(0, 1, 2, 3, 4),
     )
     def test_invalid_analytics_json_is_overwritten(
         self, project: Project, analytics_json_content: str