Fix float overflow for retry_exponential_backoff

.github/workflows/basic-tests.yml

@@ -308,6 +308,8 @@ jobs:
           --hook-stage manual
           update-build-dependencies
         if: always()
+        env:
+          SKIP_TROVE_CLASSIFIERS_ONLY: "true"
       - name: "Run automated upgrade for chart dependencies"
         run: >
           pre-commit run
@@ -372,7 +374,8 @@ jobs:
         run: breeze release-management create-minor-branch --version-branch 2-8 --answer yes
       - name: "Check Airflow RC process command"
         run: >
-          breeze release-management start-rc-process --version 2.8.3rc1 --previous-version 2.8.0 --answer yes
+          breeze release-management start-rc-process --version 2.8.3rc1 --previous-version 2.8.0
+          --sync-branch sync_v2_8_test --answer yes
       - name: "Check Airflow release process command"
         run: >
           breeze release-management start-release --release-candidate 2.8.3rc1 --previous-release 2.8.0

.github/workflows/build-images.yml

@@ -21,6 +21,10 @@ run-name: >
   Build images for ${{ github.event.pull_request.title }} ${{ github.event.pull_request._links.html.href }}
 on:  # yamllint disable-line rule:truthy
   pull_request_target:
+    branches:
+      - main
+      - v2-10-stable
+      - v2-10-test
 permissions:
   # all other permissions are set to none
   contents: read
@@ -241,14 +245,14 @@ jobs:
       pull-request-target: "true"
       is-committer-build: ${{ needs.build-info.outputs.is-committer-build }}
       push-image: "true"
-      use-uv: ${{ needs.build-info.outputs.default-branch == 'main' && 'true' || 'false' }}
+      use-uv: "true"
       image-tag: ${{ needs.build-info.outputs.image-tag }}
       platform: "linux/amd64"
       python-versions: ${{ needs.build-info.outputs.python-versions }}
       default-python-version: ${{ needs.build-info.outputs.default-python-version }}
       branch: ${{ needs.build-info.outputs.default-branch }}
       constraints-branch: ${{ needs.build-info.outputs.constraints-branch }}
-      build-provider-packages: "true"
+      build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }}
       upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
       chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }}
       docker-cache: ${{ needs.build-info.outputs.docker-cache }}

.github/workflows/ci-image-build.yml

@@ -60,7 +60,7 @@ on:  # yamllint disable-line rule:truthy
         default: "true"
         type: string
       debian-version:
-        description: "Base Debian distribution to use for the build (bookworm/bullseye)"
+        description: "Base Debian distribution to use for the build (bookworm)"
         type: string
         default: "bookworm"
       install-mysql-client-type:

.github/workflows/ci.yml

@@ -23,7 +23,7 @@ on:  # yamllint disable-line rule:truthy
   push:
     branches: ['v[0-9]+-[0-9]+-test']
   pull_request:
-    branches: ['main']
+    branches: ['main', 'v[0-9]+-[0-9]+-test', 'v[0-9]+-[0-9]+-stable']
   workflow_dispatch:
 permissions:
   # All other permissions are set to none
@@ -541,7 +541,7 @@ jobs:
       default-python-version: ${{ needs.build-info.outputs.default-python-version }}
       branch: ${{ needs.build-info.outputs.default-branch }}
       push-image: "true"
-      use-uv: ${{ needs.build-info.outputs.default-branch == 'main' && 'true' || 'false' }}
+      use-uv: "true"
       build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }}
       upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
       chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }}

.github/workflows/prod-image-build.yml

@@ -63,7 +63,7 @@ on:  # yamllint disable-line rule:truthy
         required: true
         type: string
       debian-version:
-        description: "Base Debian distribution to use for the build (bookworm/bullseye)"
+        description: "Base Debian distribution to use for the build (bookworm)"
         type: string
         default: "bookworm"
       install-mysql-client-type:

.github/workflows/prod-image-extra-checks.yml

@@ -64,27 +64,6 @@ on:  # yamllint disable-line rule:truthy
         required: true
         type: string
 jobs:
-  bullseye-image:
-    uses: ./.github/workflows/prod-image-build.yml
-    with:
-      runs-on-as-json-public: ${{ inputs.runs-on-as-json-public }}
-      build-type: "Bullseye"
-      upload-package-artifact: "false"
-      image-tag: bullseye-${{ inputs.image-tag }}
-      debian-version: "bullseye"
-      python-versions: ${{ inputs.python-versions }}
-      default-python-version: ${{ inputs.default-python-version }}
-      platform: "linux/amd64"
-      branch: ${{ inputs.branch }}
-      # Always build images during the extra checks and never push them
-      push-image: "false"
-      use-uv: ${{ inputs.use-uv }}
-      build-provider-packages: ${{ inputs.build-provider-packages }}
-      upgrade-to-newer-dependencies: ${{ inputs.upgrade-to-newer-dependencies }}
-      chicken-egg-providers: ${{ inputs.chicken-egg-providers }}
-      constraints-branch: ${{ inputs.constraints-branch }}
-      docker-cache: ${{ inputs.docker-cache }}
-
   myssql-client-image:
     uses: ./.github/workflows/prod-image-build.yml
     with:

.github/workflows/push-image-cache.yml

@@ -41,7 +41,7 @@ on:  # yamllint disable-line rule:truthy
         required: true
         type: string
       debian-version:
-        description: "Base Debian distribution to use for the build (bookworm/bullseye)"
+        description: "Base Debian distribution to use for the build (bookworm)"
         type: string
         default: "bookworm"
       install-mysql-client-type:

.github/workflows/release_dockerhub_image.yml

@@ -71,7 +71,7 @@ jobs:
   release-images:
     timeout-minutes: 120
     name: "Release images: ${{ github.event.inputs.airflowVersion }}, ${{ matrix.python-version }}"
-    runs-on: ["self-hosted", "Linux", "X64"]
+    runs-on: ["ubuntu-22.04"]
     needs: [build-info]
     strategy:
       fail-fast: false
@@ -102,15 +102,39 @@ jobs:
         run: breeze ci free-space --answer yes
       - name: "Cleanup dist and context file"
         run: rm -fv ./dist/* ./docker-context-files/*
-      - name: "Start ARM instance"
-        run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh
-        if: github.repository == 'apache/airflow'
       - name: "Login to hub.docker.com"
         run: >
           echo ${{ secrets.DOCKERHUB_TOKEN }} |
           docker login --password-stdin --username ${{ secrets.DOCKERHUB_USER }}
       - name: Login to ghcr.io
         run: echo "${{ env.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
+      - name: "Install buildx plugin"
+        # yamllint disable rule:line-length
+        run: |
+          sudo apt-get update
+          sudo apt-get install ca-certificates curl
+          sudo install -m 0755 -d /etc/apt/keyrings
+          sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc
+          sudo chmod a+r /etc/apt/keyrings/docker.asc
+
+          # Add the repository to Apt sources:
+          echo \
+          "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \
+          $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \
+          sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
+          sudo apt-get update
+          sudo apt install docker-buildx-plugin
+      - name: "Install regctl"
+        # yamllint disable rule:line-length
+        run: |
+          mkdir -p ~/bin
+          curl -L https://github.com/regclient/regclient/releases/latest/download/regctl-linux-amd64 >${HOME}/bin/regctl
+          chmod 755 ${HOME}/bin/regctl
+          echo "${HOME}/bin" >>${GITHUB_PATH}
+      - name: "Install emulation support"
+        run: docker run --privileged --rm tonistiigi/binfmt --install all
+      - name: "Create airflow_cache builder"
+        run: docker buildx create --name airflow_cache
       - name: "Prepare chicken-eggs provider packages"
         # In case of provider packages which use latest dev0 version of providers, we should prepare them
         # from the source code, not from the PyPI because they have apache-airflow>=X.Y.Z dependency
@@ -148,9 +172,6 @@ jobs:
           --limit-python ${{ matrix.python-version }} --slim-images
         env:
           COMMIT_SHA: ${{ github.sha }}
-      - name: "Stop ARM instance"
-        run: ./scripts/ci/images/ci_stop_arm_instance.sh
-        if: always() && github.repository == 'apache/airflow'
       - name: >
           Verify regular AMD64 image: ${{ github.event.inputs.airflowVersion }}, ${{ matrix.python-version }}
         run: >

.github/workflows/run-unit-tests.yml

@@ -104,6 +104,11 @@ on:  # yamllint disable-line rule:truthy
         required: false
         default: "true"
         type: string
+      database-isolation:
+        description: "Whether to enable database isolattion or not (true/false)"
+        required: false
+        default: "false"
+        type: string
       force-lowest-dependencies:
         description: "Whether to force lowest dependencies for the tests or not (true/false)"
         required: false
@@ -152,6 +157,7 @@ jobs:
       PYTHON_MAJOR_MINOR_VERSION: "${{ matrix.python-version }}"
       UPGRADE_BOTO: "${{ inputs.upgrade-boto }}"
       AIRFLOW_MONITOR_DELAY_TIME_IN_SECONDS: "${{inputs.monitor-delay-time-in-seconds}}"
+      DATABASE_ISOLATION: "${{ inputs.database-isolation }}"
       VERBOSE: "true"
     steps:
       - name: "Cleanup repo"

.github/workflows/special-tests.yml

@@ -193,6 +193,29 @@ jobs:
       run-coverage: ${{ inputs.run-coverage }}
       debug-resources: ${{ inputs.debug-resources }}
 
+  tests-database-isolation:
+    name: "Database isolation test"
+    uses: ./.github/workflows/run-unit-tests.yml
+    permissions:
+      contents: read
+      packages: read
+    secrets: inherit
+    with:
+      runs-on-as-json-default: ${{ inputs.runs-on-as-json-default }}
+      enable-aip-44: "true"
+      database-isolation: "true"
+      test-name: "DatabaseIsolation-Postgres"
+      test-scope: "DB"
+      backend: "postgres"
+      image-tag: ${{ inputs.image-tag }}
+      python-versions: "['${{ inputs.default-python-version }}']"
+      backend-versions: "['${{ inputs.default-postgres-version }}']"
+      excludes: "[]"
+      parallel-test-types-list-as-string: ${{ inputs.parallel-test-types-list-as-string }}
+      include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }}
+      run-coverage: ${{ inputs.run-coverage }}
+      debug-resources: ${{ inputs.debug-resources }}
+
   tests-quarantined:
     name: "Quarantined test"
     uses: ./.github/workflows/run-unit-tests.yml

.github/workflows/static-checks-mypy-docs.yml

@@ -243,7 +243,7 @@ jobs:
       INCLUDE_SUCCESS_OUTPUTS: "${{ inputs.include-success-outputs }}"
       PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
       VERBOSE: "true"
-    if: inputs.canary-run == 'true'
+    if: inputs.canary-run == 'true' && inputs.branch == 'main'
     steps:
       - name: "Cleanup repo"
         shell: bash

.gitignore

@@ -246,3 +246,8 @@ licenses/LICENSES-ui.txt
 
 # airflow-build-dockerfile and correconding ignore file
 airflow-build-dockerfile*
+
+# Airflow 3 files
+# These directories are ignored so someone can develop on both of them without deleting files manually
+airflow/ui
+task_sdk

.pre-commit-config.yaml

@@ -15,7 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 ---
-default_stages: [commit, push]
+default_stages: [pre-commit, pre-push]
 default_language_version:
   python: python3
   node: 22.2.0
@@ -167,6 +167,12 @@ repos:
           \.cfg$|\.conf$|\.ini$|\.ldif$|\.properties$|\.readthedocs$|\.service$|\.tf$|Dockerfile.*$
   - repo: local
     hooks:
+      - id: check-min-python-version
+        name: Check minimum Python version
+        entry: ./scripts/ci/pre_commit/check_min_python_version.py
+        language: python
+        additional_dependencies: ['rich>=12.4.4']
+        require_serial: true
       - id: update-common-sql-api-stubs
         name: Check and update common.sql API stubs
         entry: ./scripts/ci/pre_commit/update_common_sql_api_stubs.py
@@ -192,6 +198,7 @@ repos:
         files: ^.pre-commit-config.yaml$|^scripts/ci/pre_commit/update_build_dependencies.py$
         pass_filenames: false
         require_serial: true
+        additional_dependencies: ['rich>=12.4.4']
       - id: update-installers
         name: Update installers to latest (manual)
         entry: ./scripts/ci/pre_commit/update_installers.py
@@ -218,6 +225,13 @@ repos:
         files: ^airflow/models/taskinstance.py$|^airflow/models/taskinstancehistory.py$
         pass_filenames: false
         require_serial: true
+      - id: check-deferrable-default
+        name: Check and fix default value of default_deferrable
+        language: python
+        entry: ./scripts/ci/pre_commit/check_deferrable_default.py
+        pass_filenames: false
+        additional_dependencies: ["libcst>=1.1.0"]
+        files: ^airflow/.*/sensors/.*\.py$|^airflow/.*/operators/.*\.py$
   - repo: https://github.com/asottile/blacken-docs
     rev: 1.18.0
     hooks:
@@ -929,8 +943,8 @@ repos:
         name: Lint JSON Schema files with JSON Schema
         entry: ./scripts/ci/pre_commit/json_schema.py
         args:
-          - --spec-url
-          - https://json-schema.org/draft-07/schema
+          - --spec-file
+          - scripts/ci/pre_commit/draft7_schema.json
         language: python
         pass_filenames: true
         files: .*\.schema\.json$
@@ -1134,12 +1148,6 @@ repos:
         pass_filenames: true
         files: \.py$
         exclude: ^airflow/providers|^dev/.*\.py$|^scripts/.*\.py$|^tests/|^\w+_tests/|^docs/.*\.py$|^airflow/utils/helpers.py$|^hatch_build.py$
-      - id: check-deferrable-default-value
-        name: Check default value of deferrable attribute
-        language: python
-        entry: ./scripts/ci/pre_commit/check_deferrable_default.py
-        pass_filenames: false
-        files: ^airflow/.*/sensors/.*\.py$|^airflow/.*/operators/.*\.py$
       - id: check-provider-docs-valid
         name: Validate provider doc files
         entry: ./scripts/ci/pre_commit/check_provider_docs.py

Dockerfile

@@ -50,7 +50,7 @@ ARG AIRFLOW_VERSION="2.9.3"
 ARG PYTHON_BASE_IMAGE="python:3.8-slim-bookworm"
 
 ARG AIRFLOW_PIP_VERSION=24.2
-ARG AIRFLOW_UV_VERSION=0.2.33
+ARG AIRFLOW_UV_VERSION=0.4.1
 ARG AIRFLOW_USE_UV="false"
 ARG UV_HTTP_TIMEOUT="300"
 ARG AIRFLOW_IMAGE_REPOSITORY="https://github.com/apache/airflow"
@@ -124,11 +124,7 @@ function get_runtime_apt_deps() {
     echo
     echo "DEBIAN CODENAME: ${debian_version}"
     echo
-    if [[ "${debian_version}" == "bullseye" ]]; then
-        debian_version_apt_deps="libffi7 libldap-2.4-2 libssl1.1 netcat"
-    else
-        debian_version_apt_deps="libffi8 libldap-2.5-0 libssl3 netcat-openbsd"
-    fi
+    debian_version_apt_deps="libffi8 libldap-2.5-0 libssl3 netcat-openbsd"
     echo
     echo "APPLIED INSTALLATION CONFIGURATION FOR DEBIAN VERSION: ${debian_version}"
     echo
@@ -177,19 +173,6 @@ function install_debian_dev_dependencies() {
     echo
     echo "DEBIAN CODENAME: ${debian_version}"
     echo
-    if [[ "${debian_version}" == "bullseye" ]]; then
-        echo
-        echo "Bullseye detected - replacing dependencies in additional dev apt deps"
-        echo
-        # Replace dependencies in additional dev apt deps to be compatible with Bullseye
-        ADDITIONAL_DEV_APT_DEPS=${ADDITIONAL_DEV_APT_DEPS//libgcc-11-dev/libgcc-10-dev}
-        ADDITIONAL_DEV_APT_DEPS=${ADDITIONAL_DEV_APT_DEPS//netcat-openbsd/netcat}
-        echo
-        echo "Replaced bullseye dev apt dependencies"
-        echo "${ADDITIONAL_DEV_APT_COMMAND}"
-        echo
-    fi
-
     # shellcheck disable=SC2086
     apt-get install -y --no-install-recommends ${DEV_APT_DEPS} ${ADDITIONAL_DEV_APT_DEPS}
 }