From 427cf9bf97d1537f949bb16ea9782acfaacef75c Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Mon, 3 Aug 2026 04:17:10 +0200 Subject: [PATCH 1/8] Build docs and registry CI images separately so docs stay on default Python The docs build and the provider registry shared a single CI image, so they also shared its Python. That image is built at 3.12 because the registry has always extracted on 3.12, which silently moved the docs build there too and broke publishing on 2026-07-28: Sphinx mocks third-party modules while building provider docs, and from 3.12 functools copies __type_params__, for which a mock hands back another mock rather than a tuple. Any provider decorating methods with functools.wraps over a mocked callable then fails to import, and the build also began reporting cross-reference ambiguities it had not reported before. The image job already matrixes over the Python versions it is given, and every cache it touches - registry buildx tag, image stash, commit marker and mount cache - is keyed by Python version. Asking it for both versions therefore builds them in parallel, each seeding and storing its own cache, and needs no second job. Docs go back to the default Python and the registry keeps 3.12. The stashes are also named for what they are rather than for the ref alone, so release images stop sharing a key with anything else that stashes an image for the same ref. --- .github/workflows/publish-docs-to-s3.yml | 55 +++++++++++++++--------- 1 file changed, 35 insertions(+), 20 deletions(-) diff --git a/.github/workflows/publish-docs-to-s3.yml b/.github/workflows/publish-docs-to-s3.yml index f94760b64dda1..eab0ba59e9856 100644 --- a/.github/workflows/publish-docs-to-s3.yml +++ b/.github/workflows/publish-docs-to-s3.yml @@ -98,9 +98,18 @@ jobs: publish-supervisor-schema: ${{ steps.parameters.outputs.publish-supervisor-schema }} # yamllint disable rule:line-length skip-write-to-stable-folder: ${{ inputs.skip-write-to-stable-folder && '--skip-write-to-stable-folder' || '' }} - # The docs build and the registry extraction share one CI image, so they share its - # Python too. 3.12 is what the registry has always extracted on. - default-python-version: "3.12" + # The docs build and the registry extraction get an image each, built in parallel by the + # same matrix job. Sharing one image used to mean sharing its Python, which forced the docs + # onto 3.12 and broke publishing: Sphinx mocks third-party modules, and on 3.12 functools + # copies __type_params__, so mocked decorators raise TypeError and providers fail to import. + # Every image cache key already carries the Python version, so the two do not evict each other. + docs-python-version: "3.10" + registry-python-version: "3.12" + # Discriminator for every stash this workflow writes. A bare ref would collide with any + # other workflow stashing an image for the same ref, and the images here are special: built + # from a release tag rather than a branch tip, and kept for days so the next publish of that + # tag reuses them. The prefix keeps them recognisably ours. + image-stash-ref: "release-build-${{ inputs.ref }}" registry-providers: ${{ steps.derive_registry_inputs.outputs.registry-providers }} registry-full-build: ${{ steps.derive_registry_inputs.outputs.registry-full-build }} if: contains(fromJSON('[ @@ -253,8 +262,11 @@ jobs: # from these very sources, so it is the cache main's cannot be. Publishing the same ref # twice - an RC and then the final docs - skips the build outright. seed-cache-from-stashed-image: "true" - image-stash-ref: ${{ inputs.ref }} - python-versions: ${{ format('["{0}"]', needs.build-info.outputs.default-python-version) }} + image-stash-ref: ${{ needs.build-info.outputs.image-stash-ref }} + # One matrix job, two images built in parallel - docs and registry each get their own + # Python, and ci-image-build.yml keys every cache and stash it writes by Python version. + python-versions: ${{ format('["{0}", "{1}"]', needs.build-info.outputs.docs-python-version, + needs.build-info.outputs.registry-python-version) }} branch: "main" constraints-branch: "constraints-main" use-uv: "true" @@ -282,7 +294,7 @@ jobs: VERBOSE: "true" EXTRA_BUILD_OPTIONS: ${{ needs.build-info.outputs.extra-build-options }} APPLY_COMMITS: ${{ inputs.apply-commits || '' }} - PYTHON_MAJOR_MINOR_VERSION: "${{ needs.build-info.outputs.default-python-version }}" + PYTHON_MAJOR_MINOR_VERSION: "${{ needs.build-info.outputs.docs-python-version }}" DOCKER_CACHE: "registry" steps: - name: "Cleanup repo" @@ -332,24 +344,26 @@ jobs: - name: "Install Breeze from the ${{ inputs.ref }} reference" uses: ./.github/actions/breeze with: - python-version: "${{ needs.build-info.outputs.default-python-version }}" - # The image comes from the `build-ci-image` job, which builds it from this same ref and - # stashes it under the ref's own key - the shared per-branch stash holds main's image, - # not this ref's. The registry build restores the very same stash, so a run builds it - # once, and a run that had nothing to build restores what the previous publish left. + python-version: "${{ needs.build-info.outputs.docs-python-version }}" + # The image comes from the `build-ci-image` matrix, which builds it from this same ref and + # stashes it under the ref's own key - the shared per-branch stash holds main's image, not + # this ref's. The stash key carries the Python version, so this restores the docs image and + # the registry restores its own; a run that had nothing to build restores what the previous + # publish left behind. - name: > Restore CI docker image built for ref ${{ inputs.ref }} - linux/amd64:${{ needs.build-info.outputs.default-python-version }} + linux/amd64:${{ needs.build-info.outputs.docs-python-version }} uses: apache/infrastructure-actions/stash/restore@0ff9972b5872e19c9f4555c9159c2fea4f794355 with: key: "ci-image-save-v3-linux/amd64-\ - ${{ needs.build-info.outputs.default-python-version }}-${{ inputs.ref }}" + ${{ needs.build-info.outputs.docs-python-version }}\ + -${{ needs.build-info.outputs.image-stash-ref }}" path: "/mnt/" only-current-branch: 'true' fail-on-download: 'true' - - name: "Load CI image linux/amd64:${{ needs.build-info.outputs.default-python-version }}" + - name: "Load CI image linux/amd64:${{ needs.build-info.outputs.docs-python-version }}" env: - PYTHON: ${{ needs.build-info.outputs.default-python-version }} + PYTHON: ${{ needs.build-info.outputs.docs-python-version }} run: > breeze ci-image load --platform "linux/amd64" --python "${PYTHON}" --image-file-dir "/mnt" - name: "Restore docs inventory cache" @@ -511,7 +525,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_USERNAME: ${{ github.actor }} INCLUDE_SUCCESS_OUTPUTS: false - PYTHON_MAJOR_MINOR_VERSION: "${{ needs.build-info.outputs.default-python-version }}" + PYTHON_MAJOR_MINOR_VERSION: "${{ needs.build-info.outputs.docs-python-version }}" VERBOSE: "true" steps: - name: "Cleanup repo" @@ -553,7 +567,7 @@ jobs: env: AIRFLOW_VERSION: ${{ needs.build-info.outputs.airflow-version }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - PYTHON_VERSION: "${{ needs.build-info.outputs.default-python-version }}" + PYTHON_VERSION: "${{ needs.build-info.outputs.docs-python-version }}" FORCE: "true" run: > breeze sbom update-sbom-information @@ -654,10 +668,11 @@ jobs: with: destination: ${{ needs.build-info.outputs.destination }} provider: ${{ needs.build-info.outputs.registry-providers }} - python-version: ${{ needs.build-info.outputs.default-python-version }} - # `build-ci-image` already built and stashed the image this run - under the ref's key. + python-version: ${{ needs.build-info.outputs.registry-python-version }} + # `build-ci-image` already built and stashed both images this run - under the ref's key, one + # per Python. This picks the 3.12 one, which is what the registry has always extracted on. ci-image-already-built: true - image-stash-ref: ${{ inputs.ref }} + image-stash-ref: ${{ needs.build-info.outputs.image-stash-ref }} secrets: DOCS_AWS_ACCESS_KEY_ID: ${{ secrets.DOCS_AWS_ACCESS_KEY_ID }} DOCS_AWS_SECRET_ACCESS_KEY: ${{ secrets.DOCS_AWS_SECRET_ACCESS_KEY }} From 080eb2809640e8a6782382dd03958db14675be67 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Mon, 3 Aug 2026 04:25:03 +0200 Subject: [PATCH 2/8] Build each CI image in the job that uses it instead of a shared job Docs and the registry now want different Pythons, and a job that builds an image for another job has to hand it over as a multi-gigabyte tarball - exported, stashed, restored and imported - for an image whose only consumer is a single downstream job. Building in place skips that round trip entirely, and the two builds still run in parallel because both jobs now wait only on build-info. What is worth keeping between runs is the BuildKit mount cache rather than the image, so the docs job restores and stashes it under this ref and its own Python. The registry side already knew how to build its own image when the caller had not stashed one; it just could not be told which ref to build, so it would have documented main rather than the tag being published. --- .github/workflows/publish-docs-to-s3.yml | 126 ++++++++++++----------- .github/workflows/registry-build.yml | 17 +++ 2 files changed, 82 insertions(+), 61 deletions(-) diff --git a/.github/workflows/publish-docs-to-s3.yml b/.github/workflows/publish-docs-to-s3.yml index eab0ba59e9856..6e2c1f1044fdf 100644 --- a/.github/workflows/publish-docs-to-s3.yml +++ b/.github/workflows/publish-docs-to-s3.yml @@ -237,49 +237,8 @@ jobs: echo "publish-execution-api-schema=${PUBLISH_EXEC}" >> ${GITHUB_OUTPUT} echo "publish-supervisor-schema=${PUBLISH_SUP}" >> ${GITHUB_OUTPUT} - build-ci-image: - name: "Build CI image" - needs: [build-info] - uses: ./.github/workflows/ci-image-build.yml - permissions: - contents: read - packages: write - with: - runners: '["ubuntu-22.04"]' - platform: "linux/amd64" - # Built from the docs ref so the image matches the sources being documented, and - # cached against main's registry cache, which the regular Test workflow keeps warm. - checkout-ref: ${{ inputs.ref }} - push-image: "false" - upload-image-artifact: "true" - # Leaves the BuildKit mount cache behind for the next publish of this same ref; - # ci-image-build.yml always restores it, so consecutive publishes stop re-downloading - # the whole dependency set. - upload-mount-cache-artifact: "true" - # Docs are published from refs cut days or weeks before main, and main's registry cache - # stops matching at the first Dockerfile.ci change made since the cut - #70285 alone - # costs a from-scratch Python build. The image the previous publish stashed was built - # from these very sources, so it is the cache main's cannot be. Publishing the same ref - # twice - an RC and then the final docs - skips the build outright. - seed-cache-from-stashed-image: "true" - image-stash-ref: ${{ needs.build-info.outputs.image-stash-ref }} - # One matrix job, two images built in parallel - docs and registry each get their own - # Python, and ci-image-build.yml keys every cache and stash it writes by Python version. - python-versions: ${{ format('["{0}", "{1}"]', needs.build-info.outputs.docs-python-version, - needs.build-info.outputs.registry-python-version) }} - branch: "main" - constraints-branch: "constraints-main" - use-uv: "true" - upgrade-to-newer-dependencies: "false" - docker-cache: "registry" - disable-airflow-repo-cache: "false" - # Building against main's cache can fail outright when the ref's pyproject.toml has - # diverged from main — the case the docs job's inline `breeze ci-image build || docker - # buildx build` fallback used to cover. - retry-without-cache: "true" - build-docs: - needs: [build-info, build-ci-image] + needs: [build-info] timeout-minutes: 150 name: "Build documentation" runs-on: ubuntu-latest @@ -345,27 +304,69 @@ jobs: uses: ./.github/actions/breeze with: python-version: "${{ needs.build-info.outputs.docs-python-version }}" - # The image comes from the `build-ci-image` matrix, which builds it from this same ref and - # stashes it under the ref's own key - the shared per-branch stash holds main's image, not - # this ref's. The stash key carries the Python version, so this restores the docs image and - # the registry restores its own; a run that had nothing to build restores what the previous - # publish left behind. - - name: > - Restore CI docker image built for ref ${{ inputs.ref }} - linux/amd64:${{ needs.build-info.outputs.docs-python-version }} + # The image is built here rather than in a job of its own. Docs are the only consumer of + # this Python's image, so a separate job would only export a multi-gigabyte tarball and + # import it straight back. What is worth carrying between runs is the BuildKit mount cache, + # which is restored and saved below under this ref's own key. + - name: "Restore mount cache linux/amd64:${{ needs.build-info.outputs.docs-python-version }}" uses: apache/infrastructure-actions/stash/restore@0ff9972b5872e19c9f4555c9159c2fea4f794355 with: - key: "ci-image-save-v3-linux/amd64-\ + key: "ci-cache-mount-save-v3-linux/amd64-\ ${{ needs.build-info.outputs.docs-python-version }}\ -${{ needs.build-info.outputs.image-stash-ref }}" - path: "/mnt/" - only-current-branch: 'true' - fail-on-download: 'true' - - name: "Load CI image linux/amd64:${{ needs.build-info.outputs.docs-python-version }}" + path: "/tmp/" + id: restore-mount-cache + - name: "Import mount cache linux/amd64:${{ needs.build-info.outputs.docs-python-version }}" env: - PYTHON: ${{ needs.build-info.outputs.docs-python-version }} + PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.docs-python-version }} run: > - breeze ci-image load --platform "linux/amd64" --python "${PYTHON}" --image-file-dir "/mnt" + breeze ci-image import-mount-cache + --cache-file /tmp/ci-cache-mount-save-v3-${PYTHON_MAJOR_MINOR_VERSION}.tar.gz + if: steps.restore-mount-cache.outputs.stash-hit == 'true' + - name: "Login to ghcr.io" + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + ACTOR: ${{ github.actor }} + run: echo "${GITHUB_TOKEN}" | docker login ghcr.io -u ${ACTOR} --password-stdin + - name: "Build CI image linux/amd64:${{ needs.build-info.outputs.docs-python-version }}" + env: + DOCKER_CACHE: "registry" + DISABLE_AIRFLOW_REPO_CACHE: "false" + UPGRADE_TO_NEWER_DEPENDENCIES: "false" + CONSTRAINTS_GITHUB_REPOSITORY: >- + ${{ secrets.CONSTRAINTS_GITHUB_REPOSITORY != '' && + secrets.CONSTRAINTS_GITHUB_REPOSITORY || 'apache/airflow' }} + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_USERNAME: ${{ github.actor }} + PUSH: "false" + run: | + if breeze ci-image build --platform "linux/amd64"; then + exit 0 + fi + # Reached when this ref's pyproject.toml has drifted far enough from the branch the + # registry cache was built for that the cached layers cannot satisfy it. Slow, but + # correct - and the case that matters here, since docs are published from refs cut + # days or weeks before main. + echo "Build with registry cache failed - retrying with the cache disabled." + DOCKER_CACHE="disabled" breeze ci-image build --platform "linux/amd64" + - name: "Export mount cache linux/amd64:${{ needs.build-info.outputs.docs-python-version }}" + env: + PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.docs-python-version }} + run: > + breeze ci-image export-mount-cache + --cache-file /tmp/ci-cache-mount-save-v3-${PYTHON_MAJOR_MINOR_VERSION}.tar.gz + - name: "Stash mount cache linux/amd64:${{ needs.build-info.outputs.docs-python-version }}" + uses: apache/infrastructure-actions/stash/save@0ff9972b5872e19c9f4555c9159c2fea4f794355 + with: + key: "ci-cache-mount-save-v3-linux/amd64-\ + ${{ needs.build-info.outputs.docs-python-version }}\ + -${{ needs.build-info.outputs.image-stash-ref }}" + path: "/tmp/ci-cache-mount-save-v3-\ + ${{ needs.build-info.outputs.docs-python-version }}.tar.gz" + if-no-files-found: 'error' + # Read by the next publish of this same ref - an RC and then the final docs, days apart. + retention-days: '6' - name: "Restore docs inventory cache" uses: apache/infrastructure-actions/stash/restore@0ff9972b5872e19c9f4555c9159c2fea4f794355 with: @@ -658,7 +659,7 @@ jobs: update-registry: # Runs alongside the docs build rather than after it: the registry reads nothing the docs # publish produces, so waiting only added its whole duration to the release manager's wait. - needs: [build-info, build-ci-image] + needs: [build-info] if: needs.build-info.outputs.registry-providers != '' || needs.build-info.outputs.registry-full-build == 'true' name: "Update Provider Registry" permissions: @@ -668,10 +669,13 @@ jobs: with: destination: ${{ needs.build-info.outputs.destination }} provider: ${{ needs.build-info.outputs.registry-providers }} + # The registry extracts on 3.12 and the docs build on the default Python, so each builds + # the image it needs instead of sharing one. registry-build.yml builds its own whenever the + # caller has not stashed one, keyed by this ref so consecutive publishes reuse it. python-version: ${{ needs.build-info.outputs.registry-python-version }} - # `build-ci-image` already built and stashed both images this run - under the ref's key, one - # per Python. This picks the 3.12 one, which is what the registry has always extracted on. - ci-image-already-built: true + ci-image-already-built: false + # Without this the registry would document main rather than the ref being published. + checkout-ref: ${{ inputs.ref }} image-stash-ref: ${{ needs.build-info.outputs.image-stash-ref }} secrets: DOCS_AWS_ACCESS_KEY_ID: ${{ secrets.DOCS_AWS_ACCESS_KEY_ID }} diff --git a/.github/workflows/registry-build.yml b/.github/workflows/registry-build.yml index c5bae2761724d..fa8866c22a473 100644 --- a/.github/workflows/registry-build.yml +++ b/.github/workflows/registry-build.yml @@ -60,6 +60,13 @@ on: # yamllint disable-line rule:truthy required: false type: boolean default: false + checkout-ref: + description: > + Ref the CI image is built from. Defaults to the branch tip; a release publish passes the + tag it is publishing so the registry extracts from those sources rather than from main. + required: false + type: string + default: "" # `github.event_name` inside a called workflow reports the *caller's* event, never # `workflow_call`, so it cannot distinguish the two entry points. An input declared # only here can: it defaults to true when called and is undefined (falsy) on a @@ -117,10 +124,17 @@ jobs: with: runners: '["ubuntu-22.04"]' platform: "linux/amd64" + # Empty means the branch tip; a release publish passes the tag it is publishing. + checkout-ref: ${{ inputs.checkout-ref }} push-image: "false" upload-image-artifact: "true" # Kept so a standalone registry dispatch reuses the mount cache its last run left behind. upload-mount-cache-artifact: "true" + # A release ref is cut days or weeks before main, so main's registry cache stops matching at + # the first Dockerfile.ci change since the cut. What the previous publish of this same ref + # stashed was built from these very sources, so it is the cache main's cannot be. + seed-cache-from-stashed-image: ${{ inputs.image-stash-ref != '' }} + image-stash-ref: ${{ inputs.image-stash-ref }} python-versions: ${{ format('["{0}"]', inputs.python-version) }} branch: "main" constraints-branch: "constraints-main" @@ -128,6 +142,9 @@ jobs: upgrade-to-newer-dependencies: "false" docker-cache: "registry" disable-airflow-repo-cache: "false" + # Building a release ref against main's cache can fail outright once its pyproject.toml has + # diverged; falling back to an uncached build is slow but correct. + retry-without-cache: "true" build-and-publish-registry: timeout-minutes: 45 From 927ebd2905aaeccd81374e0a4c95e4588083a068 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Mon, 3 Aug 2026 04:34:20 +0200 Subject: [PATCH 3/8] Pin the docs build to the default Python again `breeze build-docs` gained a --python option, and because that option reads PYTHON_MAJOR_MINOR_VERSION, which every job of the docs publishing workflow sets, the docs quietly started building on whatever the workflow happened to export rather than on the Python they are meant for. That is how they ended up on 3.12, where Sphinx's mocking of third-party modules makes functools.wraps raise and providers fail to import. Which interpreter documents Airflow is not a per-invocation choice, so the option is gone and the build pins itself to the default again. The workflow reads the same constant instead of naming a version of its own, so the image it prepares cannot drift away from the one the docs build asks for. --- .github/workflows/publish-docs-to-s3.yml | 27 +++++++++--- dev/breeze/doc/images/output_build-docs.svg | 44 ++++++++----------- dev/breeze/doc/images/output_build-docs.txt | 2 +- .../commands/developer_commands.py | 13 +++--- .../commands/developer_commands_config.py | 1 - dev/breeze/tests/test_developer_commands.py | 33 +++++++++----- 6 files changed, 68 insertions(+), 52 deletions(-) diff --git a/.github/workflows/publish-docs-to-s3.yml b/.github/workflows/publish-docs-to-s3.yml index 6e2c1f1044fdf..ebf793282d191 100644 --- a/.github/workflows/publish-docs-to-s3.yml +++ b/.github/workflows/publish-docs-to-s3.yml @@ -98,12 +98,12 @@ jobs: publish-supervisor-schema: ${{ steps.parameters.outputs.publish-supervisor-schema }} # yamllint disable rule:line-length skip-write-to-stable-folder: ${{ inputs.skip-write-to-stable-folder && '--skip-write-to-stable-folder' || '' }} - # The docs build and the registry extraction get an image each, built in parallel by the - # same matrix job. Sharing one image used to mean sharing its Python, which forced the docs - # onto 3.12 and broke publishing: Sphinx mocks third-party modules, and on 3.12 functools - # copies __type_params__, so mocked decorators raise TypeError and providers fail to import. - # Every image cache key already carries the Python version, so the two do not evict each other. - docs-python-version: "3.10" + # Docs and the registry each build the image they need rather than sharing one. Sharing + # meant sharing the Python too, which put the docs on 3.12 and broke publishing: Sphinx + # mocks third-party modules, and from 3.12 functools copies __type_params__, so a mocked + # decorator raises TypeError and providers fail to import. `breeze build-docs` pins itself + # to the default Python, so this follows the same constant rather than restating a version. + docs-python-version: ${{ steps.default_python.outputs.default-python-version }} registry-python-version: "3.12" # Discriminator for every stash this workflow writes. A bare ref would collide with any # other workflow stashing an image for the same ref, and the images here are special: built @@ -174,6 +174,21 @@ jobs: exit 0 fi python3 dev/registry/derive_wave_providers.py + - name: "Derive the default Python version" + id: default_python + shell: bash + # `breeze build-docs` always documents on breeze's default Python, so the image this + # workflow builds for it has to be that same one. Reading the constant keeps the two from + # drifting apart and silently making the docs job build an image nothing then uses. + run: | + DEFAULT_PYTHON=$(python3 -c " + import re, pathlib + text = pathlib.Path('dev/breeze/src/airflow_breeze/global_constants.py').read_text() + versions = re.search(r'^ALL_PYTHON_MAJOR_MINOR_VERSIONS\s*=\s*\[(.*?)\]', text, re.M | re.S) + print(re.findall(r'\"([0-9]+\.[0-9]+)\"', versions.group(1))[0]) + ") + echo "Default Python version: '${DEFAULT_PYTHON}'" + echo "default-python-version=${DEFAULT_PYTHON}" >> "${GITHUB_OUTPUT}" - name: "Input parameters summary" shell: bash id: parameters diff --git a/dev/breeze/doc/images/output_build-docs.svg b/dev/breeze/doc/images/output_build-docs.svg index 78ab4748143a8..99bc5dbc0de1e 100644 --- a/dev/breeze/doc/images/output_build-docs.svg +++ b/dev/breeze/doc/images/output_build-docs.svg @@ -1,4 +1,4 @@ - +