diff --git a/.github/actions/build_ci_image_with_cache/action.yml b/.github/actions/build_ci_image_with_cache/action.yml new file mode 100644 index 0000000000000..3309891832df9 --- /dev/null +++ b/.github/actions/build_ci_image_with_cache/action.yml @@ -0,0 +1,221 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +--- +name: 'Build CI image with cache' +description: > + Builds the CI image in the job that uses it, seeding the build from what a previous run stashed + for the same ref and stashing the result for the next one. Exists so a job needing an image does + not have to take it from another job as a multi-gigabyte artifact; ci-image-build.yml remains the + reusable-workflow form for callers that do want a job of their own. +inputs: + python: + description: 'Python version to build the image for' + required: true + platform: + description: 'Platform to build for' + default: 'linux/amd64' + image-stash-ref: + description: > + Discriminator the image, its commit marker and the mount cache are stashed under. Scoped to + the ref rather than the branch: these images are built from a ref that is not the branch tip, + so the shared per-branch stash would hand the next reader sources it never asked for. + required: true + github-token: + description: 'Token used to log in to ghcr.io and read the registry build cache' + required: true + constraints-github-repository: + description: 'Repository the constraints are taken from' + default: 'apache/airflow' +runs: + using: "composite" + steps: + # A stashed image is only an answer when the sources have not moved since; otherwise it is + # cache. The commit it was built from tells the two apart and is stashed on its own, so + # deciding costs a few bytes rather than the image the decision may make unnecessary. + - name: "Restore the commit the stashed CI image was built from" + uses: apache/infrastructure-actions/stash/restore@0ff9972b5872e19c9f4555c9159c2fea4f794355 + with: + key: "ci-image-commit-v3-${{ inputs.platform }}-${{ inputs.python }}\ + -${{ inputs.image-stash-ref }}" + path: "/mnt/" + only-current-branch: 'true' + id: restore-commit + - name: "Check whether the stashed CI image was built from this commit" + id: stashed-image + env: + COMMIT_FILE: "/mnt/ci-image-commit-${{ inputs.python }}.txt" + shell: bash + run: | + head_sha="$(git rev-parse HEAD)" + stashed_sha="$(cat "${COMMIT_FILE}" 2>/dev/null || true)" + if [[ "${stashed_sha}" == "${head_sha}" ]]; then + echo "The stashed image was built from ${head_sha} - reusing it, without a build." + echo "reusable=true" >> "${GITHUB_OUTPUT}" + else + echo "The stashed image was built from '${stashed_sha:-unknown}', not ${head_sha}." + echo "It can only seed the build cache." + fi + if: steps.restore-commit.outputs.stash-hit == 'true' + # Restored ahead of the caches that feed the build, so a build made unnecessary skips them too. + - name: "Restore the CI image stashed for this ref" + uses: apache/infrastructure-actions/stash/restore@0ff9972b5872e19c9f4555c9159c2fea4f794355 + with: + key: "ci-image-save-v3-${{ inputs.platform }}-${{ inputs.python }}\ + -${{ inputs.image-stash-ref }}" + path: "/mnt/" + only-current-branch: 'true' + id: restore-image + - name: "Load the stashed CI image as the image to use" + shell: bash + env: + PLATFORM: ${{ inputs.platform }} + PYTHON: ${{ inputs.python }} + run: breeze ci-image load --platform "${PLATFORM}" --python "${PYTHON}" --image-file-dir "/mnt" + if: steps.stashed-image.outputs.reusable == 'true' + # BuildKit reads `--cache-from` from a registry, never from the local engine, so the restored + # image has to be served from one to contribute anything. + - name: "Serve the stashed CI image as build cache" + id: serve-cache + shell: bash + env: + PLATFORM: ${{ inputs.platform }} + PYTHON: ${{ inputs.python }} + CACHE_IMAGE: "localhost:5000/ci-image-cache:${{ inputs.python }}" + run: | + docker run -d --name cache-registry -p 5000:5000 registry:2 + breeze ci-image load --platform "${PLATFORM}" --python "${PYTHON}" \ + --image-file-dir "/mnt" --tag-as "${CACHE_IMAGE}" + docker push "${CACHE_IMAGE}" + echo "cache-from-image=${CACHE_IMAGE}" >> "${GITHUB_OUTPUT}" + if: > + steps.stashed-image.outputs.reusable != 'true' && + steps.restore-image.outputs.stash-hit == 'true' + # Scoped to the ref for the same reason the image is: the mount cache holds the dependency set + # the sources resolve to, and a ref's and the branch tip's are exactly what differ. + - name: "Restore the mount cache stashed for this ref" + uses: apache/infrastructure-actions/stash/restore@0ff9972b5872e19c9f4555c9159c2fea4f794355 + with: + key: "ci-cache-mount-save-v3-${{ inputs.platform }}-${{ inputs.python }}\ + -${{ inputs.image-stash-ref }}" + path: "/tmp/" + id: restore-mount-cache + if: steps.stashed-image.outputs.reusable != 'true' + - name: "Import the mount cache" + shell: bash + env: + PYTHON_MAJOR_MINOR_VERSION: ${{ inputs.python }} + CACHE_FILE: "/tmp/ci-cache-mount-save-v3-${{ inputs.python }}.tar.gz" + run: | + if [[ ! -f "${CACHE_FILE}" ]]; then + echo "${CACHE_FILE} is missing - the stash restore may have timed out. Building without it." + exit 0 + fi + breeze ci-image import-mount-cache --cache-file "${CACHE_FILE}" + if: > + steps.stashed-image.outputs.reusable != 'true' && + steps.restore-mount-cache.outputs.stash-hit == 'true' + - name: "Login to ghcr.io" + shell: bash + env: + GITHUB_TOKEN: ${{ inputs.github-token }} + ACTOR: ${{ github.actor }} + run: echo "${GITHUB_TOKEN}" | docker login ghcr.io -u "${ACTOR}" --password-stdin + if: steps.stashed-image.outputs.reusable != 'true' + - name: "Build the CI image" + shell: bash + env: + PLATFORM: ${{ inputs.platform }} + PYTHON_MAJOR_MINOR_VERSION: ${{ inputs.python }} + DOCKER_CACHE: "registry" + DISABLE_AIRFLOW_REPO_CACHE: "false" + UPGRADE_TO_NEWER_DEPENDENCIES: "false" + CONSTRAINTS_GITHUB_REPOSITORY: ${{ inputs.constraints-github-repository }} + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_TOKEN: ${{ inputs.github-token }} + GITHUB_USERNAME: ${{ github.actor }} + PUSH: "false" + VERBOSE: "true" + # Empty when nothing was stashed for this ref; breeze then builds off the registry cache + # alone. BuildKit reads --cache-from from a registry, never from the local engine. + CACHE_FROM_IMAGE: ${{ steps.serve-cache.outputs.cache-from-image }} + run: | + if breeze ci-image build --platform "${PLATFORM}"; then + exit 0 + fi + # Reached when this ref's pyproject.toml has drifted far enough from the branch the + # registry cache was built for that the cached layers cannot satisfy it. Slow, but correct + # - and the case that matters here, since these refs are cut days or weeks before main. + echo "Build with the registry cache failed - retrying with the cache disabled." + DOCKER_CACHE="disabled" breeze ci-image build --platform "${PLATFORM}" + if: steps.stashed-image.outputs.reusable != 'true' + - name: "Stop serving the stashed image as cache" + # The registry holds a second copy of a multi-gigabyte image and the export below needs room. + shell: bash + env: + CACHE_FROM_IMAGE: ${{ steps.serve-cache.outputs.cache-from-image }} + run: | + docker rm -f cache-registry + docker rmi "${CACHE_FROM_IMAGE}" + if: always() && steps.serve-cache.outputs.cache-from-image != '' + - name: "Export the CI image for the next run" + shell: bash + env: + PLATFORM: ${{ inputs.platform }} + COMMIT_FILE: "/mnt/ci-image-commit-${{ inputs.python }}.txt" + run: | + breeze ci-image save --platform "${PLATFORM}" --image-file-dir "/mnt" + git rev-parse HEAD > "${COMMIT_FILE}" + if: steps.stashed-image.outputs.reusable != 'true' + - name: "Stash the CI image" + uses: apache/infrastructure-actions/stash/save@0ff9972b5872e19c9f4555c9159c2fea4f794355 + with: + key: "ci-image-save-v3-${{ inputs.platform }}-${{ inputs.python }}\ + -${{ inputs.image-stash-ref }}" + path: "/mnt/ci-image-save-*-${{ inputs.python }}.tar" + if-no-files-found: 'error' + # Read by the next run for this same ref - an RC and then the final, days apart. + retention-days: '6' + if: steps.stashed-image.outputs.reusable != 'true' + # Saved last and with the image's retention, so that finding this commit is enough to know the + # image it describes is there to be restored. + - name: "Stash the commit the CI image was built from" + uses: apache/infrastructure-actions/stash/save@0ff9972b5872e19c9f4555c9159c2fea4f794355 + with: + key: "ci-image-commit-v3-${{ inputs.platform }}-${{ inputs.python }}\ + -${{ inputs.image-stash-ref }}" + path: "/mnt/ci-image-commit-${{ inputs.python }}.txt" + if-no-files-found: 'error' + retention-days: '6' + if: steps.stashed-image.outputs.reusable != 'true' + - name: "Export the mount cache for the next run" + shell: bash + env: + PYTHON_MAJOR_MINOR_VERSION: ${{ inputs.python }} + run: > + breeze ci-image export-mount-cache + --cache-file /tmp/ci-cache-mount-save-v3-${PYTHON_MAJOR_MINOR_VERSION}.tar.gz + if: steps.stashed-image.outputs.reusable != 'true' + - name: "Stash the mount cache" + uses: apache/infrastructure-actions/stash/save@0ff9972b5872e19c9f4555c9159c2fea4f794355 + with: + key: "ci-cache-mount-save-v3-${{ inputs.platform }}-${{ inputs.python }}\ + -${{ inputs.image-stash-ref }}" + path: "/tmp/ci-cache-mount-save-v3-${{ inputs.python }}.tar.gz" + if-no-files-found: 'error' + retention-days: '6' + if: steps.stashed-image.outputs.reusable != 'true' diff --git a/.github/workflows/publish-docs-to-s3.yml b/.github/workflows/publish-docs-to-s3.yml index f94760b64dda1..1c613209794b4 100644 --- a/.github/workflows/publish-docs-to-s3.yml +++ b/.github/workflows/publish-docs-to-s3.yml @@ -98,9 +98,18 @@ jobs: publish-supervisor-schema: ${{ steps.parameters.outputs.publish-supervisor-schema }} # yamllint disable rule:line-length skip-write-to-stable-folder: ${{ inputs.skip-write-to-stable-folder && '--skip-write-to-stable-folder' || '' }} - # The docs build and the registry extraction share one CI image, so they share its - # Python too. 3.12 is what the registry has always extracted on. - default-python-version: "3.12" + # Docs and the registry each build the image they need rather than sharing one. Sharing + # meant sharing the Python too, which put the docs on 3.12 and broke publishing: Sphinx + # mocks third-party modules, and from 3.12 functools copies __type_params__, so a mocked + # decorator raises TypeError and providers fail to import. `breeze build-docs` pins itself + # to the default Python, so this follows the same constant rather than restating a version. + docs-python-version: ${{ steps.default_python.outputs.default-python-version }} + registry-python-version: "3.12" + # Discriminator for every stash this workflow writes. A bare ref would collide with any + # other workflow stashing an image for the same ref, and the images here are special: built + # from a release tag rather than a branch tip, and kept for days so the next publish of that + # tag reuses them. Naming them after this workflow keeps them recognisably ours. + image-stash-ref: "publish-docs-${{ inputs.ref }}" registry-providers: ${{ steps.derive_registry_inputs.outputs.registry-providers }} registry-full-build: ${{ steps.derive_registry_inputs.outputs.registry-full-build }} if: contains(fromJSON('[ @@ -165,6 +174,21 @@ jobs: exit 0 fi python3 dev/registry/derive_wave_providers.py + - name: "Derive the default Python version" + id: default_python + shell: bash + # `breeze build-docs` always documents on breeze's default Python, so the image this + # workflow builds for it has to be that same one. Reading the constant keeps the two from + # drifting apart and silently making the docs job build an image nothing then uses. + run: | + DEFAULT_PYTHON=$(python3 -c " + import re, pathlib + text = pathlib.Path('dev/breeze/src/airflow_breeze/global_constants.py').read_text() + versions = re.search(r'^ALL_PYTHON_MAJOR_MINOR_VERSIONS\s*=\s*\[(.*?)\]', text, re.M | re.S) + print(re.findall(r'\"([0-9]+\.[0-9]+)\"', versions.group(1))[0]) + ") + echo "Default Python version: '${DEFAULT_PYTHON}'" + echo "default-python-version=${DEFAULT_PYTHON}" >> "${GITHUB_OUTPUT}" - name: "Input parameters summary" shell: bash id: parameters @@ -228,46 +252,8 @@ jobs: echo "publish-execution-api-schema=${PUBLISH_EXEC}" >> ${GITHUB_OUTPUT} echo "publish-supervisor-schema=${PUBLISH_SUP}" >> ${GITHUB_OUTPUT} - build-ci-image: - name: "Build CI image" - needs: [build-info] - uses: ./.github/workflows/ci-image-build.yml - permissions: - contents: read - packages: write - with: - runners: '["ubuntu-22.04"]' - platform: "linux/amd64" - # Built from the docs ref so the image matches the sources being documented, and - # cached against main's registry cache, which the regular Test workflow keeps warm. - checkout-ref: ${{ inputs.ref }} - push-image: "false" - upload-image-artifact: "true" - # Leaves the BuildKit mount cache behind for the next publish of this same ref; - # ci-image-build.yml always restores it, so consecutive publishes stop re-downloading - # the whole dependency set. - upload-mount-cache-artifact: "true" - # Docs are published from refs cut days or weeks before main, and main's registry cache - # stops matching at the first Dockerfile.ci change made since the cut - #70285 alone - # costs a from-scratch Python build. The image the previous publish stashed was built - # from these very sources, so it is the cache main's cannot be. Publishing the same ref - # twice - an RC and then the final docs - skips the build outright. - seed-cache-from-stashed-image: "true" - image-stash-ref: ${{ inputs.ref }} - python-versions: ${{ format('["{0}"]', needs.build-info.outputs.default-python-version) }} - branch: "main" - constraints-branch: "constraints-main" - use-uv: "true" - upgrade-to-newer-dependencies: "false" - docker-cache: "registry" - disable-airflow-repo-cache: "false" - # Building against main's cache can fail outright when the ref's pyproject.toml has - # diverged from main — the case the docs job's inline `breeze ci-image build || docker - # buildx build` fallback used to cover. - retry-without-cache: "true" - build-docs: - needs: [build-info, build-ci-image] + needs: [build-info] timeout-minutes: 150 name: "Build documentation" runs-on: ubuntu-latest @@ -282,7 +268,7 @@ jobs: VERBOSE: "true" EXTRA_BUILD_OPTIONS: ${{ needs.build-info.outputs.extra-build-options }} APPLY_COMMITS: ${{ inputs.apply-commits || '' }} - PYTHON_MAJOR_MINOR_VERSION: "${{ needs.build-info.outputs.default-python-version }}" + PYTHON_MAJOR_MINOR_VERSION: "${{ needs.build-info.outputs.docs-python-version }}" DOCKER_CACHE: "registry" steps: - name: "Cleanup repo" @@ -313,6 +299,16 @@ jobs: ref: ${{ inputs.ref }} fetch-tags: true fetch-depth: 0 + # A local action is read from the workspace, not from the ref the workflow itself runs from, + # so an action added after this tag was cut is simply absent. Checking the workflow's own + # version out alongside supplies the definition; the image is still built from the sources + # above. It has to come after the checkout that populates the root, which cleans untracked + # siblings away. + - name: "Checkout the workflow version for its actions" + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + path: workflow-version - name: "Apply patch commits if provided" run: | if [[ "${APPLY_COMMITS}" != "" ]]; then @@ -332,26 +328,21 @@ jobs: - name: "Install Breeze from the ${{ inputs.ref }} reference" uses: ./.github/actions/breeze with: - python-version: "${{ needs.build-info.outputs.default-python-version }}" - # The image comes from the `build-ci-image` job, which builds it from this same ref and - # stashes it under the ref's own key - the shared per-branch stash holds main's image, - # not this ref's. The registry build restores the very same stash, so a run builds it - # once, and a run that had nothing to build restores what the previous publish left. - - name: > - Restore CI docker image built for ref ${{ inputs.ref }} - linux/amd64:${{ needs.build-info.outputs.default-python-version }} - uses: apache/infrastructure-actions/stash/restore@0ff9972b5872e19c9f4555c9159c2fea4f794355 + python-version: "${{ needs.build-info.outputs.docs-python-version }}" + # Built here rather than in a job of its own: a separate job could only hand the image over + # as a multi-gigabyte artifact, for a single consumer. The image, the commit it was built + # from and the mount cache are still stashed under this ref, so the next publish of the same + # ref seeds its build from them - or skips the build outright when the sources have not moved. + - name: "Build CI image linux/amd64:${{ needs.build-info.outputs.docs-python-version }}" + uses: ./workflow-version/.github/actions/build_ci_image_with_cache with: - key: "ci-image-save-v3-linux/amd64-\ - ${{ needs.build-info.outputs.default-python-version }}-${{ inputs.ref }}" - path: "/mnt/" - only-current-branch: 'true' - fail-on-download: 'true' - - name: "Load CI image linux/amd64:${{ needs.build-info.outputs.default-python-version }}" - env: - PYTHON: ${{ needs.build-info.outputs.default-python-version }} - run: > - breeze ci-image load --platform "linux/amd64" --python "${PYTHON}" --image-file-dir "/mnt" + python: "${{ needs.build-info.outputs.docs-python-version }}" + platform: "linux/amd64" + image-stash-ref: "${{ needs.build-info.outputs.image-stash-ref }}" + github-token: "${{ secrets.GITHUB_TOKEN }}" + constraints-github-repository: >- + ${{ secrets.CONSTRAINTS_GITHUB_REPOSITORY != '' && + secrets.CONSTRAINTS_GITHUB_REPOSITORY || 'apache/airflow' }} - name: "Restore docs inventory cache" uses: apache/infrastructure-actions/stash/restore@0ff9972b5872e19c9f4555c9159c2fea4f794355 with: @@ -499,7 +490,14 @@ jobs: overwrite: 'true' publish-docs-to-s3: - needs: [build-docs, build-java-sdk-docs, build-info] + needs: [build-docs, build-java-sdk-docs, build-info, update-registry] + # `update-registry` is skipped for non-provider distributions, and a job whose dependency was + # skipped is skipped in turn, so its result is checked rather than implied - which means the + # two that are not conditional have to be asserted here as well. + if: > + !cancelled() && needs.build-docs.result == 'success' && + needs.build-java-sdk-docs.result == 'success' && + needs.update-registry.result != 'failure' name: "Publish documentation to S3" permissions: id-token: write @@ -511,7 +509,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_USERNAME: ${{ github.actor }} INCLUDE_SUCCESS_OUTPUTS: false - PYTHON_MAJOR_MINOR_VERSION: "${{ needs.build-info.outputs.default-python-version }}" + PYTHON_MAJOR_MINOR_VERSION: "${{ needs.build-info.outputs.docs-python-version }}" VERBOSE: "true" steps: - name: "Cleanup repo" @@ -553,7 +551,7 @@ jobs: env: AIRFLOW_VERSION: ${{ needs.build-info.outputs.airflow-version }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - PYTHON_VERSION: "${{ needs.build-info.outputs.default-python-version }}" + PYTHON_VERSION: "${{ needs.build-info.outputs.docs-python-version }}" FORCE: "true" run: > breeze sbom update-sbom-information @@ -643,8 +641,9 @@ jobs: update-registry: # Runs alongside the docs build rather than after it: the registry reads nothing the docs - # publish produces, so waiting only added its whole duration to the release manager's wait. - needs: [build-info, build-ci-image] + # build produces, so ordering the two only added its whole duration to the wait. The publish + # does wait for it, so a run reports success once both halves are actually out. + needs: [build-info] if: needs.build-info.outputs.registry-providers != '' || needs.build-info.outputs.registry-full-build == 'true' name: "Update Provider Registry" permissions: @@ -654,10 +653,12 @@ jobs: with: destination: ${{ needs.build-info.outputs.destination }} provider: ${{ needs.build-info.outputs.registry-providers }} - python-version: ${{ needs.build-info.outputs.default-python-version }} - # `build-ci-image` already built and stashed the image this run - under the ref's key. - ci-image-already-built: true - image-stash-ref: ${{ inputs.ref }} + # The registry extracts on 3.12 and the docs build on the default Python, so each builds the + # image it needs in the job that uses it, keyed by this ref so consecutive publishes reuse it. + python-version: ${{ needs.build-info.outputs.registry-python-version }} + # Without this the registry would read main rather than the ref being published. + checkout-ref: ${{ inputs.ref }} + image-stash-ref: ${{ needs.build-info.outputs.image-stash-ref }} secrets: DOCS_AWS_ACCESS_KEY_ID: ${{ secrets.DOCS_AWS_ACCESS_KEY_ID }} DOCS_AWS_SECRET_ACCESS_KEY: ${{ secrets.DOCS_AWS_SECRET_ACCESS_KEY }} diff --git a/.github/workflows/registry-build.yml b/.github/workflows/registry-build.yml index c5bae2761724d..da239882629a1 100644 --- a/.github/workflows/registry-build.yml +++ b/.github/workflows/registry-build.yml @@ -55,11 +55,13 @@ on: # yamllint disable-line rule:truthy required: false type: string default: "3.12" - ci-image-already-built: - description: "Caller has already built and stashed the CI image for this run" + checkout-ref: + description: > + Ref the registry is built from. Defaults to the branch tip; a release publish passes the + tag it is publishing so the extraction reads those sources rather than main's. required: false - type: boolean - default: false + type: string + default: "" # `github.event_name` inside a called workflow reports the *caller's* event, never # `workflow_call`, so it cannot distinguish the two entry points. An input declared # only here can: it defaults to true when called and is undefined (falsy) on a @@ -88,57 +90,13 @@ permissions: packages: read jobs: - build-ci-image: - name: "Build CI image" - uses: ./.github/workflows/ci-image-build.yml - permissions: - contents: read - packages: write - # Skipped when the caller stashed an image for this run; a standalone dispatch has no - # such image and still builds its own. - if: > - (inputs.ci-image-already-built != true) && ( - inputs.is-workflow-call || - contains(fromJSON('[ - "ashb", - "bugraoz93", - "eladkal", - "ephraimbuddy", - "jedcunningham", - "jscheffl", - "kaxil", - "pierrejeambrun", - "shahar1", - "potiuk", - "uranusjr", - "utkarsharma2", - "vincbeck" - ]'), github.event.sender.login)) - with: - runners: '["ubuntu-22.04"]' - platform: "linux/amd64" - push-image: "false" - upload-image-artifact: "true" - # Kept so a standalone registry dispatch reuses the mount cache its last run left behind. - upload-mount-cache-artifact: "true" - python-versions: ${{ format('["{0}"]', inputs.python-version) }} - branch: "main" - constraints-branch: "constraints-main" - use-uv: "true" - upgrade-to-newer-dependencies: "false" - docker-cache: "registry" - disable-airflow-repo-cache: "false" - build-and-publish-registry: timeout-minutes: 45 name: "Build & Publish Registry" - needs: [build-ci-image] - # `build-ci-image` is skipped when the caller stashed the image, so this cannot simply - # inherit its result. On a dispatch it enforces the committer allowlist below; on the - # `workflow_call` path the caller has already gated on its own allowlist, so this must - # not re-check it against a second, separately-maintained copy. + # On a dispatch this enforces the committer allowlist below; on the `workflow_call` path the + # caller has already gated on its own allowlist, so this must not re-check it against a + # second, separately-maintained copy. if: > - !cancelled() && needs.build-ci-image.result != 'failure' && ( inputs.is-workflow-call || contains(fromJSON('[ "ashb", @@ -154,7 +112,7 @@ jobs: "uranusjr", "utkarsharma2", "vincbeck" - ]'), github.event.sender.login)) + ]'), github.event.sender.login) runs-on: ubuntu-latest env: SCARF_ANALYTICS: "false" @@ -169,25 +127,51 @@ jobs: REGISTRY_CACHE_CONTROL: public, max-age=300 permissions: contents: read + # The image is built here now, so this job reads the ghcr registry build cache itself. + packages: read steps: - name: "Checkout repository" uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + # Empty means the branch tip; a release publish passes the tag it is publishing, so the + # extraction reads the sources being released rather than whatever main holds. + ref: ${{ inputs.checkout-ref }} # Tags drive the phantom-version filter in extract_metadata.py # (only versions with a real `providers-/` tag are # treated as released). Without this, the filter silently # falls back to `versions[0]` and ships phantom versions. fetch-tags: true - - name: "Prepare breeze & CI image" - uses: ./.github/actions/prepare_breeze_and_image + # A local action is read from the workspace, not from the ref the workflow itself runs from, + # so an action added after the checked-out ref was cut is simply absent. Checking the + # workflow's own version out alongside supplies the definition; the image is still built + # from the sources checked out above. + - name: "Checkout the workflow version for its actions" + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + path: workflow-version + + - name: "Make /mnt writeable and cleanup" + run: ./scripts/ci/make_mnt_writeable.sh + - name: "Free up disk space" + run: ./scripts/tools/free_up_disk_space.sh + - name: "Install Breeze" + uses: ./.github/actions/breeze + with: + python-version: "${{ inputs.python-version }}" + + # Built in this job rather than handed over by one of its own: the image has a single + # consumer, so a separate job could only export it and import it straight back. It is still + # stashed under this ref, so the next run for the same ref seeds its build from it. + - name: "Build CI image linux/amd64:${{ inputs.python-version }}" + uses: ./workflow-version/.github/actions/build_ci_image_with_cache with: python: "${{ inputs.python-version }}" platform: "linux/amd64" - use-uv: "true" - make-mnt-writeable-and-cleanup: "true" image-stash-ref: "${{ inputs.image-stash-ref }}" + github-token: "${{ secrets.GITHUB_TOKEN }}" - name: "Install AWS CLI v2" run: | diff --git a/dev/breeze/doc/images/output_build-docs.svg b/dev/breeze/doc/images/output_build-docs.svg index 78ab4748143a8..99bc5dbc0de1e 100644 --- a/dev/breeze/doc/images/output_build-docs.svg +++ b/dev/breeze/doc/images/output_build-docs.svg @@ -1,4 +1,4 @@ - +