diff --git a/.github/workflows/ci-image-build.yml b/.github/workflows/ci-image-build.yml index 7f1fa61fe609e..980c9c7a22399 100644 --- a/.github/workflows/ci-image-build.yml +++ b/.github/workflows/ci-image-build.yml @@ -29,6 +29,11 @@ on: # yamllint disable-line rule:truthy required: false default: "" type: string + checkout-ref: + description: "Commit-ish to checkout sources from (empty = workflow ref)." + required: false + default: "" + type: string pull-request-target: description: "Whether we are running this from pull-request-target workflow (true/false)" required: false @@ -120,6 +125,7 @@ jobs: - name: "Checkout target branch" uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: + ref: ${{ inputs.checkout-ref }} persist-credentials: false - name: "Free up disk space" shell: bash diff --git a/.github/workflows/generate-constraints.yml b/.github/workflows/generate-constraints.yml index 55604039f7734..2a8d8f4f72dd8 100644 --- a/.github/workflows/generate-constraints.yml +++ b/.github/workflows/generate-constraints.yml @@ -52,6 +52,11 @@ on: # yamllint disable-line rule:truthy description: "Whether to use uvloop (true/false)" required: true type: string + checkout-ref: + description: "Commit-ish to checkout sources from (empty = workflow ref)." + required: false + default: "" + type: string jobs: generate-constraints-matrix: permissions: @@ -76,9 +81,10 @@ jobs: - name: "Cleanup repo" shell: bash run: sudo rm -rf ${GITHUB_WORKSPACE}/* - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + - name: "Checkout ${{ inputs.checkout-ref || github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: + ref: ${{ inputs.checkout-ref }} persist-credentials: false - name: "Install prek" uses: ./.github/actions/install-prek diff --git a/.github/workflows/update-constraints-on-push.yml b/.github/workflows/update-constraints-on-push.yml index 16460bd89cbb0..b65168c53dae1 100644 --- a/.github/workflows/update-constraints-on-push.yml +++ b/.github/workflows/update-constraints-on-push.yml @@ -16,7 +16,20 @@ # under the License. # --- -name: Update constraints on push for main (only when uv.lock changes) +name: Update constraints (on uv.lock push or manual dispatch) +# This workflow refreshes the pinned constraint files stored in the +# `constraints-*` branches. It runs automatically whenever `uv.lock` changes on +# `main` or a `vX-Y-test` branch, and can also be triggered manually via the +# "Run workflow" button (workflow_dispatch) - for example to pick up newly +# released providers/dependencies from PyPI just before promoting an RC. +# +# The manual run is always launched from `main` (so the workflow definition and +# `breeze` come from `main`), and the `ref` input selects the commit-ish +# (branch, tag or commit hash) whose sources the constraints are refreshed from. +# The `constraints-X-Y` branch to push to is derived from that ref's +# `dev/breeze/src/airflow_breeze/branch_defaults.py`, so no cherry-pick to the +# `vX-Y-test` / `vX-Y-stable` branch is needed. +# See dev/MANUALLY_GENERATING_IMAGE_CACHE_AND_CONSTRAINTS.md for details. on: # yamllint disable-line rule:truthy push: branches: @@ -24,11 +37,27 @@ on: # yamllint disable-line rule:truthy - v[0-9]+-[0-9]+-test paths: - 'uv.lock' + workflow_dispatch: + inputs: + ref: + description: >- + Commit-ish to refresh constraints from (branch, tag or commit hash), + e.g. `v3-3-test`, `v3-3-stable`, `constraints-3-3` or a tag/hash. + The matching `constraints-X-Y` branch is derived from that ref. + required: true + type: string + upgrade-to-newer-dependencies: + description: >- + Re-resolve to the newest matching dependencies (picks up newly + released providers/dependencies from PyPI). Leave enabled when + refreshing constraints before promoting an RC. + type: boolean + default: true permissions: contents: read concurrency: - group: ${{ github.workflow }}-${{ github.ref }} + group: ${{ github.workflow }}-${{ github.ref }}-${{ inputs.ref }} cancel-in-progress: true env: GITHUB_REPOSITORY: ${{ github.repository }} @@ -39,6 +68,25 @@ jobs: build-info: name: "Build info" runs-on: ["ubuntu-22.04"] + # Automatic uv.lock-push runs are unrestricted; manual (workflow_dispatch) + # runs are limited to release managers (same allowlist as the prod image + # release workflow) because they push to the protected constraints-* branches. + if: >- + github.event_name != 'workflow_dispatch' || + contains(fromJSON('[ + "ashb", + "bugraoz93", + "eladkal", + "ephraimbuddy", + "jedcunningham", + "jscheffl", + "kaxil", + "pierrejeambrun", + "potiuk", + "utkarsharma2", + "vincbeck", + "vatsrahul1001", + ]'), github.event.sender.login) outputs: default-branch: ${{ steps.selective-checks.outputs.default-branch }} default-constraints-branch: ${{ steps.selective-checks.outputs.default-constraints-branch }} @@ -49,14 +97,10 @@ jobs: - name: "Cleanup repo" shell: bash run: sudo rm -rf ${GITHUB_WORKSPACE}/* - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - with: - persist-credentials: false - - name: Fetch incoming commit ${{ github.sha }} with its parent + - name: "Checkout ${{ inputs.ref || github.ref }} ( ${{ inputs.ref || github.sha }} )" uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: - ref: ${{ github.sha }} + ref: ${{ inputs.ref || github.sha }} fetch-depth: 2 persist-credentials: false - name: "Install Breeze" @@ -75,7 +119,7 @@ jobs: id: selective-checks env: PR_LABELS: "[]" - COMMIT_REF: "${{ github.sha }}" + COMMIT_REF: "${{ inputs.ref || github.sha }}" VERBOSE: "false" GITHUB_CONTEXT_INPUT: "${{ runner.temp }}/github_context.json" run: breeze ci selective-check 2>> ${GITHUB_OUTPUT} @@ -96,8 +140,11 @@ jobs: python-versions: ${{ needs.build-info.outputs.python-versions }} branch: ${{ needs.build-info.outputs.default-branch }} constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} + checkout-ref: ${{ inputs.ref }} use-uv: "true" - upgrade-to-newer-dependencies: "false" + upgrade-to-newer-dependencies: >- + ${{ github.event_name == 'workflow_dispatch' + && inputs.upgrade-to-newer-dependencies && 'true' || 'false' }} docker-cache: "registry" disable-airflow-repo-cache: "false" @@ -113,6 +160,7 @@ jobs: generate-pypi-constraints: "true" generate-no-providers-constraints: "true" debug-resources: "false" + checkout-ref: ${{ inputs.ref }} use-uv: "true" update-constraints: @@ -125,6 +173,7 @@ jobs: packages: read env: PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }} + CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }} steps: - name: "Cleanup repo" shell: bash @@ -133,14 +182,11 @@ jobs: uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - - name: "Set constraints branch name" - id: constraints-branch - run: ./scripts/ci/constraints/ci_branch_constraints.sh >> ${GITHUB_OUTPUT} - - name: Checkout ${{ steps.constraints-branch.outputs.branch }} + - name: Checkout ${{ env.CONSTRAINTS_BRANCH }} uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: path: "constraints" - ref: ${{ steps.constraints-branch.outputs.branch }} + ref: ${{ env.CONSTRAINTS_BRANCH }} persist-credentials: true fetch-depth: 0 - name: "Download constraints from the generate-constraints job" diff --git a/dev/MANUALLY_GENERATING_IMAGE_CACHE_AND_CONSTRAINTS.md b/dev/MANUALLY_GENERATING_IMAGE_CACHE_AND_CONSTRAINTS.md index 68104fee02d33..4a794333ab664 100644 --- a/dev/MANUALLY_GENERATING_IMAGE_CACHE_AND_CONSTRAINTS.md +++ b/dev/MANUALLY_GENERATING_IMAGE_CACHE_AND_CONSTRAINTS.md @@ -31,6 +31,7 @@ - [What the command does](#what-the-command-does) - [Manually generating constraint files](#manually-generating-constraint-files) - [Why we need to generate constraint files manually](#why-we-need-to-generate-constraint-files-manually) + - [How to refresh constraints via the CI workflow (recommended)](#how-to-refresh-constraints-via-the-ci-workflow-recommended) - [How to generate constraint files](#how-to-generate-constraint-files) - [Is it safe to generate constraints manually?](#is-it-safe-to-generate-constraints-manually) - [Manually updating already tagged constraint files](#manually-updating-already-tagged-constraint-files) @@ -213,6 +214,54 @@ with tests, but we KNOW that the tip of the branch is good and we want to releas we want to move the PRs of contributors to start using the new constraints. This should be done with caution and you need to be sure what you are doing, but you can always do it manually if you want. +## How to refresh constraints via the CI workflow (recommended) + +The easiest way to refresh the constraints - for example to pick up newly released +providers/dependencies from PyPI just before promoting an RC - is to trigger the +[`Update constraints`](../.github/workflows/update-constraints-on-push.yml) workflow manually +instead of running the `breeze` commands locally. The workflow runs exactly the same steps +that run automatically when `uv.lock` changes, builds the CI images, generates all constraint +flavours and commits/pushes them to the matching `constraints-*` branch. + +The manual run is always launched from `main` and takes a `ref` input that selects the +commit-ish (branch, tag or commit hash) whose sources the constraints are refreshed from. You +do **not** select the release branch in GitHub's branch dropdown, and you do **not** need to +cherry-pick anything to the `vX-Y-test` / `vX-Y-stable` branch first - the workflow definition +and `breeze` come from `main`, while the sources come from the `ref` you pass. The only +requirement is that the `breeze` constraint-generation commands work against that `ref` (they +are stable across recent branches). + +Manual runs are restricted to release managers (the same allowlist as the prod image release +workflow); the automatic `uv.lock`-push runs are not restricted. + +To run it: + +1. Go to the [`Update constraints`](https://github.com/apache/airflow/actions/workflows/update-constraints-on-push.yml) + workflow in the Actions tab. +2. Click **Run workflow** and keep the branch set to `main` (this is where the workflow runs + from - it is not the branch whose constraints get refreshed). +3. In the **ref** field, enter the commit-ish to refresh constraints from - for example + `v3-3-test`, `v3-3-stable`, `constraints-3-3`, an RC tag, or a commit hash. The matching + `constraints-X-Y` branch to push to is derived automatically from that ref's + `dev/breeze/src/airflow_breeze/branch_defaults.py`, so pointing at anything on the 3.3 line + refreshes `constraints-3-3`. +4. Keep **Re-resolve to the newest matching dependencies** enabled (the default) so the run + picks up the latest released providers/dependencies from PyPI. Disable it only if you want + to regenerate constraints strictly from that ref's `uv.lock` without upgrading. +5. Once the run finishes, verify the new commit on the matching constraints branch + (for example `constraints-3-3` for a 3.3 refresh): + + https://github.com/apache/airflow/commits/constraints--/ + +> [!NOTE] +> Because `v3-3-test` and `v3-3-stable` can diverge while RCs are being voted on (fixes are +> cherry-picked to `v3-3-test`), be deliberate about which `ref` you refresh from. Refresh from +> the ref that matches the artifacts you are about to promote - typically `v3-3-stable` (or the +> RC tag) for a release, not `v3-3-test`. + +If you cannot or do not want to use the workflow, you can still generate the constraints +locally with the `breeze` commands below. + ## How to generate constraint files ```bash diff --git a/dev/README_RELEASE_AIRFLOW.md b/dev/README_RELEASE_AIRFLOW.md index f7d8dc7961d55..7e6005e75bba0 100644 --- a/dev/README_RELEASE_AIRFLOW.md +++ b/dev/README_RELEASE_AIRFLOW.md @@ -1372,6 +1372,16 @@ breeze release-management start-release \ Note: The `--task-sdk-version` parameter is optional. If you are releasing Airflow without a corresponding Task SDK release, you can omit this parameter. +Note: When it reaches the constraints step, `start-release` asks whether to base the final +`constraints-${VERSION}` tag on the latest `constraints-X-Y` branch tip instead of the RC +constraints tag. The RC constraints are frozen when the RC is cut, so if any providers were +released (or constraints were otherwise refreshed - see +[MANUALLY_GENERATING_IMAGE_CACHE_AND_CONSTRAINTS.md](MANUALLY_GENERATING_IMAGE_CACHE_AND_CONSTRAINTS.md)) +after the last RC and you want the released constraints to reflect that, answer **yes** to tag the +`constraints-X-Y` branch tip. Otherwise (the default) the final tag matches the RC exactly. If you +do refresh, run the `Update constraints` workflow from `main` with `ref` set to the ref you are +releasing (typically `v3-*-stable`) **before** running `start-release`. + 4. Make sure to update Airflow version in ``v3-*-test`` branch after cherry-picking to X.Y.1 in ``airflow/__init__.py`` diff --git a/dev/breeze/src/airflow_breeze/commands/release_command.py b/dev/breeze/src/airflow_breeze/commands/release_command.py index 5b183e765d965..1b93f9c28ac43 100644 --- a/dev/breeze/src/airflow_breeze/commands/release_command.py +++ b/dev/breeze/src/airflow_breeze/commands/release_command.py @@ -343,10 +343,27 @@ def upload_to_pypi(version, task_sdk_version=None): ) +def get_constraints_branch_for_version(version: str) -> str: + major, minor = version.split(".")[:2] + return f"constraints-{major}-{minor}" + + def retag_constraints(release_candidate, version): - if confirm_action(f"Retag constraints for {release_candidate} as {version}?"): + # By default the final ``constraints-`` tag is created from the RC constraints tag. + # If the constraints were refreshed after the last RC (e.g. to pick up newly released + # providers - see dev/MANUALLY_GENERATING_IMAGE_CACHE_AND_CONSTRAINTS.md) the tip of the + # ``constraints-X-Y`` branch is newer than the RC tag and should be tagged instead. + constraints_branch = get_constraints_branch_for_version(version) + source_ref = f"constraints-{release_candidate}" + if confirm_action( + f"Base the final constraints on the latest '{constraints_branch}' branch tip instead of the " + f"'{source_ref}' tag? Choose yes if you refreshed constraints after {release_candidate}." + ): + run_command(["git", "fetch", "origin", constraints_branch], check=True) + source_ref = f"origin/{constraints_branch}" + if confirm_action(f"Retag constraints from {source_ref} as {version}?"): run_command( - ["git", "checkout", f"constraints-{release_candidate}"], + ["git", "checkout", source_ref], check=True, ) run_command( diff --git a/dev/breeze/tests/test_release_command.py b/dev/breeze/tests/test_release_command.py index 2745a8387eb05..1e023e6bf5bfd 100644 --- a/dev/breeze/tests/test_release_command.py +++ b/dev/breeze/tests/test_release_command.py @@ -538,3 +538,55 @@ def fake_getcwd() -> str: assert "task-sdk" not in " ".join(console_messages).lower() assert run_command_calls == [] assert chdir_calls == [svn_release_repo, "/original/dir"] + + +@pytest.mark.parametrize( + ("version", "expected_branch"), + [ + ("3.3.0", "constraints-3-3"), + ("3.0.5", "constraints-3-0"), + ("3.12.10", "constraints-3-12"), + ], +) +def test_get_constraints_branch_for_version(release_cmd, version, expected_branch): + assert release_cmd.get_constraints_branch_for_version(version) == expected_branch + + +def test_retag_constraints_from_rc_tag_by_default(monkeypatch, release_cmd): + run_command_calls: list[list[str]] = [] + + def fake_confirm_action(prompt: str, **_kwargs) -> bool: + # Decline basing on the branch tip; accept the retag and the push. + return not prompt.startswith("Base the final constraints on the latest") + + monkeypatch.setattr(release_cmd, "confirm_action", fake_confirm_action) + monkeypatch.setattr(release_cmd, "run_command", lambda cmd, **_kwargs: run_command_calls.append(cmd)) + + release_cmd.retag_constraints("3.3.0rc2", "3.3.0") + + assert ["git", "checkout", "constraints-3.3.0rc2"] in run_command_calls + assert ["git", "tag", "-s", "constraints-3.3.0", "-m", "Constraints for Apache Airflow 3.3.0"] in ( + run_command_calls + ) + assert ["git", "push", "origin", "tag", "constraints-3.3.0"] in run_command_calls + # The branch tip is not fetched when basing on the RC tag. + assert not any(cmd[:2] == ["git", "fetch"] for cmd in run_command_calls) + + +def test_retag_constraints_from_branch_tip_when_confirmed(monkeypatch, release_cmd): + run_command_calls: list[list[str]] = [] + + def fake_confirm_action(prompt: str, **_kwargs) -> bool: + return True + + monkeypatch.setattr(release_cmd, "confirm_action", fake_confirm_action) + monkeypatch.setattr(release_cmd, "run_command", lambda cmd, **_kwargs: run_command_calls.append(cmd)) + + release_cmd.retag_constraints("3.3.0rc2", "3.3.0") + + assert ["git", "fetch", "origin", "constraints-3-3"] in run_command_calls + assert ["git", "checkout", "origin/constraints-3-3"] in run_command_calls + assert ["git", "checkout", "constraints-3.3.0rc2"] not in run_command_calls + assert ["git", "tag", "-s", "constraints-3.3.0", "-m", "Constraints for Apache Airflow 3.3.0"] in ( + run_command_calls + )