test: Improve use of test fixtures in integration tests

.github/workflows/python-package.yml

@@ -18,7 +18,7 @@ on:
         - '.github/workflows/python-package.yml'
 
 jobs:
-  build:
+  Unit:
 
     runs-on: ubuntu-20.04
     strategy:
@@ -50,10 +50,11 @@ jobs:
           ${{ matrix.python-version }}-v1-
     - name: Install dependencies
       run: |
-        python -m pip install --upgrade pip
-        python -m pip install flake8 pytest case pytest-celery pytest-subtests pytest-timeout pytest-cov
-        python -m pip install moto boto3 msgpack PyYAML
-        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+        python -m pip install -r requirements/dev.txt \
+            -r requirements/test.txt \
+            -r requirements/test-ci-default.txt \
+            -r requirements/docs.txt \
+            -r requirements/pkgutils.txt
 
     - name: Run Unit test with pytest
       run: |
@@ -71,3 +72,71 @@ jobs:
             flags: unittests # optional
             fail_ci_if_error: true # optional (default = false)
             verbose: true # optional (default = false)
+
+
+  Integration:
+
+    runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ['3.7', '3.8', '3.9', 'pypy3']
+
+     # additional service containers to run
+    services:
+      # postgres service
+      rabbitmq:
+        # docker hub image
+        image: rabbitmq
+        # configure the instance
+        env:
+          RABBITMQ_DEFAULT_USER: craiga
+          RABBITMQ_DEFAULT_PASS: security_is_important
+        ports:
+          - 5432:5432
+
+      # redis service
+      redis:
+        # Docker Hub image
+        image: redis
+        ports:
+        - 6379:6379
+        # configure the instance
+        env:
+        # The hostname used to communicate with the Redis service container
+          REDIS_HOST: localhost
+          # The default Redis port
+          REDIS_PORT: 6379
+
+    steps:
+    - name: Install apt packages
+      run: |
+        sudo apt update && sudo apt install -f libcurl4-openssl-dev libssl-dev gnutls-dev httping expect libmemcached-dev
+    - uses: actions/checkout@v2
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Get pip cache dir
+      id: pip-cache
+      run: |
+        echo "::set-output name=dir::$(pip cache dir)"
+    - name: Cache
+      uses: actions/cache@v2
+      with:
+        path: ${{ steps.pip-cache.outputs.dir }}
+        key:
+          ${{ matrix.python-version }}-v1-${{ hashFiles('**/setup.py') }}
+        restore-keys: |
+          ${{ matrix.python-version }}-v1-
+    - name: Install dependencies
+      run: |
+        python -m pip install -r requirements/dev.txt \
+            -r requirements/test.txt \
+            -r requirements/test-ci-default.txt \
+            -r requirements/test-integration.txt
+
+    - name: Run Integration test with pytest / tox /tox-docker
+      run: |
+        PYTHONPATH=. pytest -xsv t/integration

celery/contrib/pytest.py

@@ -90,6 +90,7 @@ def celery_session_worker(
             celery_session_app.tasks.register(class_task)
         with worker.start_worker(celery_session_app,
                                  pool=celery_worker_pool,
+                                 shutdown_timeout=60.0,
                                  **celery_worker_parameters) as w:
             yield w
 

requirements/test-ci-base.txt

@@ -1,5 +1,4 @@
 pytest-cov
-pytest-travis-fold
 codecov
 -r extras/redis.txt
 -r extras/sqlalchemy.txt

requirements/test-ci-default.txt

@@ -12,7 +12,7 @@
 -r extras/thread.txt
 -r extras/elasticsearch.txt
 -r extras/couchdb.txt
--r extras/couchbase.txt
+#-r extras/couchbase.txt
 -r extras/arangodb.txt
 -r extras/consul.txt
 -r extras/cosmosdbsql.txt

t/integration/conftest.py

@@ -1,4 +1,6 @@
+import itertools
 import os
+import uuid
 
 import pytest
 
@@ -42,6 +44,45 @@ def celery_config():
     }
 
 
+@pytest.fixture(scope="session")
+def requires_redis_backend(celery_config):
+    if not celery_config["result_backend"].startswith("redis://"):
+        raise pytest.skip("Skipping tests requiring a Redis backend")
+
+
+@pytest.fixture(scope="function")
+def redis_connection(requires_redis_backend):
+    return get_redis_connection()
+
+
+@pytest.fixture(scope="function")
+def unique_redis_key(redis_connection):
+    redis_key = str(uuid.uuid4())
+    redis_connection.delete(redis_key)
+    yield redis_key
+    redis_connection.delete(redis_key)
+
+
+@pytest.fixture(scope="function")
+def unique_redis_keys(redis_connection):
+    def gen_redis_keys():
+        while True:
+            key_to_yield = str(uuid.uuid4())
+            redis_connection.delete(key_to_yield)
+            yield key_to_yield
+
+    key_gen = gen_redis_keys()
+    to_yield, for_teardown = itertools.tee(key_gen, 2)
+    yield to_yield
+
+    try:
+        key_gen.throw(GeneratorExit)
+    except GeneratorExit:
+        pass
+    for redis_key in for_teardown:
+        redis_connection.delete(redis_key)
+
+
 @pytest.fixture(scope='session')
 def celery_enable_logging():
     return True
@@ -67,6 +108,14 @@ def manager(app, celery_session_worker):
     return Manager(app)
 
 
+@pytest.fixture(scope="function")
+def requires_chord_support(manager):
+    try:
+        manager.app.backend.ensure_chords_allowed()
+    except NotImplementedError as e:
+        raise pytest.skip("Skipping test requiring support for chords")
+
+
 @pytest.fixture(autouse=True)
 def ZZZZ_set_app_current(app):
     app.set_current()