Merge branch 'main' into beat/shutdown-signals

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 5.3.4
+current_version = 5.4.0
 commit = True
 tag = True
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(?P<releaselevel>[a-z\d]+)?

.github/ISSUE_TEMPLATE/Bug-Report.md

@@ -29,6 +29,7 @@ To check an item on the list replace [ ] with [x].
   to find out if the bug was already fixed in the main branch.
 - [ ] I have included all related issues and possible duplicate issues
   in this issue (If there are none, check this box anyway).
+- [ ] I have tried to reproduce the issue with [pytest-celery](https://docs.celeryq.dev/projects/pytest-celery/en/latest/userguide/celery-bug-report.html) and added the reproduction script below.
 
 ## Mandatory Debugging Information
 
@@ -137,6 +138,10 @@ We prefer submitting test cases in the form of a PR to our integration test suit
 If you can provide one, please mention the PR number below.
 If not, please attach the most minimal code example required to reproduce the issue below.
 If the test case is too large, please include a link to a gist or a repository below.
+
+Alternatively, the pytest-celery plugin can be used to create standalone reproduction scripts
+that can be added to this report. See the pytest-celery documentation for more information at
+pytest-celery.readthedocs.io
 -->
 
 <details>

.github/ISSUE_TEMPLATE/Enhancement.md

@@ -22,7 +22,7 @@ To check an item on the list replace [ ] with [x].
 - [ ] I have checked the [pull requests list](https://github.com/celery/celery/pulls?q=is%3Apr+label%3A%22Issue+Type%3A+Enhancement%22+-label%3A%22Category%3A+Documentation%22)
   for existing proposed enhancements.
 - [ ] I have checked the [commit log](https://github.com/celery/celery/commits/main)
-  to find out if the if the same enhancement was already implemented in the
+  to find out if the same enhancement was already implemented in the
   main branch.
 - [ ] I have included all related issues and possible duplicate issues in this issue
       (If there are none, check this box anyway).

.github/ISSUE_TEMPLATE/Minor-Version-Release-Checklist.md

@@ -7,19 +7,130 @@ assignees: ''
 
 ---
 
-Version: <!-- Insert Version Here -->
-Release PR: <!-- Insert Release PR Here -->
+# Minor Release Overview: v<!-- Insert Version Here, e.g. 5.4.0 -->
 
-# Checklist
+This issue will summarize the status and discussion in preparation for the new release. It will be used to track the progress of the release and to ensure that all the necessary steps are taken. It will serve as a checklist for the release and will be used to communicate the status of the release to the community.
 
-- [ ] Release PR drafted
-- [ ] Release PR reviewed
-- [ ] The main branch build passes
+> ⚠️ **Warning:** The release checklist is a living document. It will be updated as the release progresses. Please check back often to ensure that you are up to date with the latest information.
 
-    [![Build Status](https://github.com/celery/celery/actions/workflows/python-package.yml/badge.svg)](https://github.com/celery/celery/actions/workflows/python-package.yml)
-- [ ] Release Notes
-- [ ] What's New
+## Checklist
+- [ ] Codebase Stability
+- [ ] Breaking Changes Validation
+- [ ] Compile Changelog
+- [ ] Release
+- [ ] Release Announcement
+
+# Release Details
+The release manager is responsible for completing the release end-to-end ensuring that all the necessary steps are taken and that the release is completed in a timely manner. This is usually the owner of the release issue but may be assigned to a different maintainer if necessary.
+
+- Release Manager: <!-- Insert Release Manager Here -->
+- Release Date: <!-- Insert Release Date Here -->
+- Release Branch: `main`
 
-# Release Blockers
+# Release Steps
+The release manager is expected to execute the checklist below. The release manager is also responsible for ensuring that the checklist is updated as the release progresses. Any changes or issues should be communicated under this issue for centralized tracking.
 
 # Potential Release Blockers
+
+## 1. Codebase Stability
+- [ ] The `main` branch build passes
+
+    [![Build Status](https://github.com/celery/celery/actions/workflows/python-package.yml/badge.svg)](https://github.com/celery/celery/actions/workflows/python-package.yml)
+
+## 2. Breaking Changes Validation
+A patch release should not contain any breaking changes. The release manager is responsible for reviewing all of the merged PRs since the last release to ensure that there are no breaking changes. If there are any breaking changes, the release manager should discuss with the maintainers to determine the best course of action if an obvious solution is not apparent.
+
+## 3. Compile Changelog
+The release changelog is set in two different places:
+1. The [Changelog.rst](https://github.com/celery/celery/blob/main/Changelog.rst) that uses the RST format.
+2. The GitHub Release auto-generated changelog that uses the Markdown format. This is auto-generated by the GitHub Draft Release UI.
+
+> ⚠️ **Warning:** The pre-commit changes should not be included in the changelog.
+
+To generate the changelog automatically, [draft a new release](https://github.com/celery/celery/releases/new) on GitHub using a fake new version tag for the automatic changelog generation. Notice the actual tag creation is done **on publish** so we can use that to generate the changelog and then delete the draft release without publishing it thus avoiding creating a new tag.
+
+- Create a new tag
+<img width="502" alt="CleanShot 2023-09-05 at 22 06 24@2x" src="https://github.com/celery/celery/assets/4662342/69d7eb78-0dd1-4a37-9266-d718e2340fad">
+
+- Generate Markdown release notes
+<img width="389" alt="CleanShot 2023-09-05 at 22 13 39@2x" src="https://github.com/celery/celery/assets/4662342/83c6a4b6-1676-496e-8d75-5a6b10352b5b">
+
+- Copy the generated release notes.
+
+- Delete the draft release without publishing it.
+
+### 3.1 Changelog.rst
+Once you have the actual changes, you need to convert it to rst format and add it to the [Changelog.rst](https://github.com/celery/celery/blob/main/Changelog.rst) file. The new version block needs to follow the following format:
+```rst
+.. _version-x.y.z:
+
+x.y.z
+=====
+
+:release-date: YYYY-MM-DD HH:MM P.M/A.M TimeZone
+:release-by: Release Manager Name
+
+Changes list in RST format.
+```
+
+These changes will reflect in the [Change history](https://docs.celeryq.dev/en/stable/changelog.html) section of the documentation.
+
+### 3.2 Changelog PR
+The changes to the [Changelog.rst](https://github.com/celery/celery/blob/main/Changelog.rst) file should be submitted as a PR. This will PR should be the last merged PR before the release.
+
+## 4. Release
+### 4.1 Prepare releasing environment
+Before moving forward with the release, the release manager should ensure that bumpversion and twine are installed. These are required to publish the release.
+
+### 4.2 Bump version
+The release manager should bump the version using the following command:
+```bash
+bumpversion patch
+```
+The changes should be pushed directly to main by the release manager.
+
+At this point, the git log should appear somewhat similar to this:
+```
+commit XXX (HEAD -> main, tag: vX.Y.Z, upstream/main, origin/main)
+Author: Release Manager
+Date:   YYY
+
+    Bump version: a.b.c → x.y.z
+
+commit XXX
+Author: Release Manager
+Date:   YYY
+
+    Added changelog for vX.Y.Z (#1234)
+```
+If everything looks good, the bump version commit can be directly pushed to `main`:
+```bash
+git push origin main --tags
+```
+
+### 4.3 Publish release to PyPI
+The release manager should publish the release to PyPI using the following commands running under the root directory of the repository:
+```bash
+python setup.py clean build sdist bdist_wheel
+```
+If the build is successful, the release manager should publish the release to PyPI using the following command:
+```bash
+twine upload dist/celery-X.Y.Z*
+```
+
+> ⚠️ **Warning:** The release manager should double check that the release details are correct (project/version) before publishing the release to PyPI.
+
+> ⚠️ **Critical Reminder:** Should the released package prove to be faulty or need retraction for any reason, do not delete it from PyPI. The appropriate course of action is to "yank" the release.
+
+## Release Announcement
+After the release is published, the release manager should create a new GitHub Release and set it as the latest release.
+
+<img width="401" alt="CleanShot 2023-09-05 at 22 51 24@2x" src="https://github.com/celery/celery/assets/4662342/6edcba33-b6a9-4242-85e8-8260a2ede868">
+
+### Add Release Notes
+On a per-case basis, the release manager may also attach an additional release note to the auto-generated release notes. This is usually done when there are important changes that are not reflected in the auto-generated release notes.
+
+### OpenCollective Update
+After successfully publishing the new release, the release manager is responsible for announcing it on the project's OpenCollective [page](https://opencollective.com/celery/updates). This is to engage with the community and keep backers and sponsors in the loop.
+
+

.github/ISSUE_TEMPLATE/config.yml

@@ -9,3 +9,5 @@ contact_links:
   - name: py-amqp Issue Tracker
     url: https://github.com/celery/py-amqp/issues/
     about: If this issue only involves py-amqp, please open a new issue there.
+  - name: pytest-celery Issue Tracker
+    url: https://github.com/celery/pytest-celery/issues/

.github/dependabot.yml

@@ -4,3 +4,7 @@ updates:
     directory: "/"
     schedule:
       interval: "daily"
+  - package-ecosystem: "pip"
+    directory: "/"
+    schedule:
+      interval: "daily"

.github/workflows/codeql-analysis.yml

@@ -17,7 +17,9 @@ on:
   pull_request:
     # The branches below must be a subset of the branches above
     branches: [ main ]
-
+  workflow_dispatch:
+
+
 
 jobs:
   analyze:
@@ -41,7 +43,7 @@ jobs:
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v2
+      uses: github/codeql-action/init@v3
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -52,7 +54,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v2
+      uses: github/codeql-action/autobuild@v3
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl
@@ -66,4 +68,4 @@ jobs:
     #   make release
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v2
+      uses: github/codeql-action/analyze@v3

.github/workflows/docker.yml

@@ -1,29 +1,71 @@
 name: Docker
 
 on:
-    push:
+    pull_request:
       branches: [ 'main']
       paths:
           - '**.py'
           - '**.txt'
           - '**.toml'
-          - './docker/**'
+          - '/docker/**'
           - '.github/workflows/docker.yml'
-    pull_request:
+          - 'Dockerfile'
+    push:
       branches: [ 'main']
       paths:
           - '**.py'
           - '**.txt'
           - '**.toml'
-          - './docker/**'
+          - '/docker/**'
           - '.github/workflows/docker.yml'
+    workflow_dispatch:
 
 
 jobs:
   docker-build:
     runs-on: ubuntu-latest
-    timeout-minutes: 30
+    timeout-minutes: 60
     steps:
     - uses: actions/checkout@v4
     - name: Build Docker container
-      run: make docker-build
+      run: make docker-build
+
+  docker-docs:
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+    - uses: actions/checkout@v4
+    - name: Build Documentation
+      run: make docker-docs
+
+  smoke-tests_dev:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+    - uses: actions/checkout@v4
+    - name: "Build smoke tests container: dev"
+      run: docker build -f t/smoke/workers/docker/dev .
+
+  smoke-tests_latest:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+    - uses: actions/checkout@v4
+    - name: "Build smoke tests container: latest"
+      run: docker build -f t/smoke/workers/docker/pypi .
+
+  smoke-tests_pypi:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+    - uses: actions/checkout@v4
+    - name: "Build smoke tests container: pypi"
+      run: docker build -f t/smoke/workers/docker/pypi --build-arg CELERY_VERSION="5" .
+
+  smoke-tests_legacy:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+    - uses: actions/checkout@v4
+    - name: "Build smoke tests container: legacy"
+      run: docker build -f t/smoke/workers/docker/pypi --build-arg CELERY_VERSION="4" .

.github/workflows/linter.yml

@@ -1,6 +1,6 @@
 name: Linter
 
-on: [pull_request]
+on: [pull_request, workflow_dispatch]
 
 jobs:
   linter:
@@ -11,4 +11,4 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Run pre-commit
-        uses: pre-commit/action@v3.0.0
+        uses: pre-commit/action@v3.0.1