Merge branch 'celery:main' into main

.github/workflows/docker.yml

@@ -0,0 +1,29 @@
+name: Docker
+
+on:
+    push:
+      branches: [ 'main']
+      paths:
+          - '**.py'
+          - '**.txt'
+          - '**.toml'
+          - './docker/**'
+          - '.github/workflows/docker.yml'
+    pull_request:
+      branches: [ 'main']
+      paths:
+          - '**.py'
+          - '**.txt'
+          - '**.toml'
+          - './docker/**'
+          - '.github/workflows/docker.yml'
+
+
+jobs:
+  docker-build:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+    - uses: actions/checkout@v4
+    - name: Build Docker container
+      run: make docker-build

Makefile

@@ -53,6 +53,12 @@ help:
 	@echo "bump-minor           - Bump minor version number."
 	@echo "bump-major           - Bump major version number."
 	@echo "release              - Make PyPI release."
+	@echo ""
+	@echo "Docker-specific commands:"
+	@echo "  docker-build			- Build celery docker container."
+	@echo "  docker-lint        		- Run tox -e lint on docker container."
+	@echo "  docker-unit-tests		- Run unit tests on docker container, use '-- -k <TEST NAME>' for specific test run."
+	@echo "  docker-bash        		- Get a bash shell inside the container."
 
 clean: clean-docs clean-pyc clean-build
 
@@ -167,3 +173,30 @@ graph: clean-graph $(WORKER_GRAPH)
 
 authorcheck:
 	git shortlog -se | cut -f2 | extra/release/attribution.py
+
+.PHONY: docker-build
+docker-build:
+	@docker-compose -f docker/docker-compose.yml build
+
+.PHONY: docker-lint
+docker-lint:
+	@docker-compose -f docker/docker-compose.yml run --rm -w /home/developer/celery celery tox -e lint
+
+.PHONY: docker-unit-tests
+docker-unit-tests:
+	@docker-compose -f docker/docker-compose.yml run --rm -w /home/developer/celery celery tox -e 3.11-unit -- $(filter-out $@,$(MAKECMDGOALS))
+
+# Integration tests are not fully supported when running in a docker container yet so we allow them to
+# gracefully fail until fully supported.
+# TODO: Add documentation (in help command) when fully supported.
+.PHONY: docker-integration-tests
+docker-integration-tests:
+	@docker-compose -f docker/docker-compose.yml run --rm -w /home/developer/celery celery tox -e 3.11-integration-docker -- --maxfail=1000
+
+.PHONY: docker-bash
+docker-bash:
+	@docker-compose -f docker/docker-compose.yml run --rm -w /home/developer/celery celery bash
+
+.PHONY: catch-all
+%: catch-all
+	@:

docker/Dockerfile

@@ -74,7 +74,7 @@ COPY --chown=1000:1000 docker/entrypoint /entrypoint
 RUN chmod gu+x /entrypoint
 
 # Define the local pyenvs
-RUN pyenv local 3.8 3.9 3.10 3.11
+RUN pyenv local 3.11 3.10 3.9 3.8
 
 RUN pyenv exec python3.8 -m pip install --upgrade pip setuptools wheel && \
     pyenv exec python3.9 -m pip install --upgrade pip setuptools wheel && \
@@ -119,19 +119,21 @@ RUN pyenv exec python3.8 -m pip install \
   -r requirements/test-integration.txt \
   -r requirements/test-pypy3.txt \
   -r requirements/test.txt && \
-    pyenv exec python3.11 -m pip install \
-    -r requirements/default.txt \
-    -r requirements/dev.txt \
-    -r requirements/docs.txt \
-    -r requirements/pkgutils.txt \
-    -r requirements/test-ci-base.txt \
-    -r requirements/test-ci-default.txt \
-    -r requirements/test-integration.txt \
-    -r requirements/test-pypy3.txt \
-    -r requirements/test.txt
+  pyenv exec python3.11 -m pip install \
+  -r requirements/default.txt \
+  -r requirements/dev.txt \
+  -r requirements/docs.txt \
+  -r requirements/pkgutils.txt \
+  -r requirements/test-ci-base.txt \
+  -r requirements/test-ci-default.txt \
+  -r requirements/test-integration.txt \
+  -r requirements/test-pypy3.txt \
+  -r requirements/test.txt
 
 WORKDIR $HOME/celery
 
+RUN git config --global --add safe.directory /home/developer/celery
+
 # Setup the entrypoint, this ensures pyenv is initialized when a container is started
 # and that any compiled files from earlier steps or from mounts are removed to avoid
 # pytest failing with an ImportMismatchError

docs/userguide/concurrency/gevent.rst

@@ -0,0 +1,79 @@
+.. _concurrency-eventlet:
+
+===========================
+ Concurrency with gevent
+===========================
+
+.. _gevent-introduction:
+
+Introduction
+============
+
+The `gevent`_ homepage describes it a coroutine_ -based Python_ networking library that uses
+`greenlet <https://greenlet.readthedocs.io>`_ to provide a high-level synchronous API on top of the `libev`_
+or `libuv`_ event loop.
+
+Features include:
+
+* Fast event loop based on `libev`_ or `libuv`_.
+* Lightweight execution units based on greenlets.
+* API that re-uses concepts from the Python standard library (for
+  examples there are `events`_ and
+  `queues`_).
+* `Cooperative sockets with SSL support <http://www.gevent.org/api/index.html#networking>`_
+* `Cooperative DNS queries <http://www.gevent.org/dns.html>`_ performed through a threadpool,
+  dnspython, or c-ares.
+* `Monkey patching utility <http://www.gevent.org/intro.html#monkey-patching>`_ to get 3rd party modules to become cooperative
+* TCP/UDP/HTTP servers
+* Subprocess support (through `gevent.subprocess`_)
+* Thread pools
+
+gevent is `inspired by eventlet`_ but features a more consistent API,
+simpler implementation and better performance. Read why others `use
+gevent`_ and check out the list of the `open source projects based on
+gevent`_.
+
+
+Enabling gevent
+=================
+
+You can enable the gevent pool by using the
+:option:`celery worker -P gevent` or  :option:`celery worker --pool=gevent`
+worker option.
+
+.. code-block:: console
+
+    $ celery -A proj worker -P gevent -c 1000
+
+.. _eventlet-examples:
+
+Examples
+========
+
+See the `gevent examples`_ directory in the Celery distribution for
+some examples taking use of Eventlet support.
+
+Known issues
+============
+There is a known issue using python 3.11 and gevent.
+The issue is documented `here`_ and addressed in a `gevent issue`_.
+Upgrading to greenlet 3.0 solves it.
+
+.. _events: http://www.gevent.org/api/gevent.event.html#gevent.event.Event
+.. _queues: http://www.gevent.org/api/gevent.queue.html#gevent.queue.Queue
+.. _`gevent`: http://www.gevent.org/
+.. _`gevent examples`:
+    https://github.com/celery/celery/tree/main/examples/gevent
+.. _gevent.subprocess: http://www.gevent.org/api/gevent.subprocess.html#module-gevent.subprocess
+
+.. _coroutine: https://en.wikipedia.org/wiki/Coroutine
+.. _Python: http://python.org
+.. _libev: http://software.schmorp.de/pkg/libev.html
+.. _libuv: http://libuv.org
+.. _inspired by eventlet: http://blog.gevent.org/2010/02/27/why-gevent/
+.. _use gevent: http://groups.google.com/group/gevent/browse_thread/thread/4de9703e5dca8271
+.. _open source projects based on gevent: https://github.com/gevent/gevent/wiki/Projects
+.. _what's new: http://www.gevent.org/whatsnew_1_5.html
+.. _changelog: http://www.gevent.org/changelog.html
+.. _here: https://github.com/celery/celery/issues/8425
+.. _gevent issue: https://github.com/gevent/gevent/issues/1985

docs/userguide/concurrency/index.rst

@@ -11,3 +11,4 @@
     :maxdepth: 2
 
     eventlet
+    gevent

examples/gevent/README.rst

@@ -0,0 +1,51 @@
+==================================
+  Example using the gevent Pool
+==================================
+
+Introduction
+============
+
+This is a Celery application containing two example tasks.
+
+First you need to install gevent::
+
+    $ python -m pip install gevent celery pybloom-live
+
+Before you run any of the example tasks you need to start
+the worker::
+
+    $ cd examples/gevent
+    $ celery worker -l INFO --concurrency=500 --pool=gevent
+
+As usual you need to have RabbitMQ running, see the Celery getting started
+guide if you haven't installed it yet.
+
+Tasks
+=====
+
+* `tasks.urlopen`
+
+This task simply makes a request opening the URL and returns the size
+of the response body::
+
+    $ cd examples/gevent
+    $ python
+    >>> from tasks import urlopen
+    >>> urlopen.delay('https://www.google.com/').get()
+    9980
+
+To open several URLs at once you can do::
+
+    $ cd examples/gevent
+    $ python
+    >>> from tasks import urlopen
+    >>> from celery import group
+    >>> result = group(urlopen.s(url)
+    ...                     for url in LIST_OF_URLS).apply_async()
+    >>> for incoming_result in result.iter_native():
+    ...     print(incoming_result)
+
+
+This is a simple recursive web crawler.  It will only crawl
+URLs for the current host name.  Please see comments in the
+`webcrawler.py` file.

requirements/test.txt

@@ -1,4 +1,4 @@
-pytest==7.4.1
+pytest==7.4.2
 pytest-celery==0.0.0
 pytest-subtests==0.11.0
 pytest-timeout==2.1.0
@@ -8,7 +8,7 @@ boto3>=1.26.143
 moto>=4.1.11
 # typing extensions
 mypy==1.5.1; platform_python_implementation=="CPython"
-pre-commit==3.3.3
+pre-commit==3.4.0
 -r extras/yaml.txt
 -r extras/msgpack.txt
 -r extras/mongodb.txt

t/integration/test_canvas.py

@@ -3500,6 +3500,6 @@ def on_signature(self, sig, **headers) -> dict:
             canvas.options["link_error"] = dict(fail.si())
             canvas.stamp(visitor=CustomStampingVisitor())
 
-        with subtests.test(msg='Expect canvas to fail'):
+        with subtests.test(msg="Expect canvas to fail"):
             with pytest.raises(ExpectedException):
                 canvas.apply_async().get(timeout=TIMEOUT)

tox.ini

@@ -3,7 +3,7 @@ requires =
     tox-gh-actions
 envlist =
     {3.8,3.9,3.10,3.11,pypy3}-unit
-    {3.8,3.9,3.10,3.11,pypy3}-integration-{rabbitmq_redis,rabbitmq,redis,dynamodb,azureblockblob,cache,cassandra,elasticsearch}
+    {3.8,3.9,3.10,3.11,pypy3}-integration-{rabbitmq_redis,rabbitmq,redis,dynamodb,azureblockblob,cache,cassandra,elasticsearch,docker}
 
     flake8
     apicheck
@@ -66,6 +66,9 @@ setenv =
     rabbitmq_redis: TEST_BROKER=pyamqp://
     rabbitmq_redis: TEST_BACKEND=redis://
 
+    docker: TEST_BROKER=pyamqp://rabbit:5672
+    docker: TEST_BACKEND=redis://redis
+
     dynamodb: TEST_BROKER=redis://
     dynamodb: TEST_BACKEND=dynamodb://@localhost:8000
     dynamodb: AWS_ACCESS_KEY_ID=test_aws_key_id
@@ -81,7 +84,7 @@ basepython =
     3.11: python3.11
     pypy3: pypy3
     mypy: python3.8
-    lint,apicheck,linkcheck,configcheck,bandit: python3.9
+    lint,apicheck,linkcheck,configcheck,bandit: python3.11
 usedevelop = True
 
 [testenv:mypy]