Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions .github/workflows/ci-image-build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,11 @@ on: # yamllint disable-line rule:truthy
required: false
default: ""
type: string
checkout-ref:
description: "Commit-ish to checkout sources from (empty = workflow ref)."
required: false
default: ""
type: string
pull-request-target:
description: "Whether we are running this from pull-request-target workflow (true/false)"
required: false
Expand Down Expand Up @@ -120,6 +125,7 @@ jobs:
- name: "Checkout target branch"
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
ref: ${{ inputs.checkout-ref }}
persist-credentials: false
- name: "Free up disk space"
shell: bash
Expand Down
8 changes: 7 additions & 1 deletion .github/workflows/generate-constraints.yml
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,11 @@ on: # yamllint disable-line rule:truthy
description: "Whether to use uvloop (true/false)"
required: true
type: string
checkout-ref:
description: "Commit-ish to checkout sources from (empty = workflow ref)."
required: false
default: ""
type: string
jobs:
generate-constraints-matrix:
permissions:
Expand All @@ -76,9 +81,10 @@ jobs:
- name: "Cleanup repo"
shell: bash
run: sudo rm -rf ${GITHUB_WORKSPACE}/*
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- name: "Checkout ${{ inputs.checkout-ref || github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
ref: ${{ inputs.checkout-ref }}
persist-credentials: false
- name: "Install prek"
uses: ./.github/actions/install-prek
Expand Down
76 changes: 61 additions & 15 deletions .github/workflows/update-constraints-on-push.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,19 +16,48 @@
# under the License.
#
---
name: Update constraints on push for main (only when uv.lock changes)
name: Update constraints (on uv.lock push or manual dispatch)
# This workflow refreshes the pinned constraint files stored in the
# `constraints-*` branches. It runs automatically whenever `uv.lock` changes on
# `main` or a `vX-Y-test` branch, and can also be triggered manually via the
# "Run workflow" button (workflow_dispatch) - for example to pick up newly
# released providers/dependencies from PyPI just before promoting an RC.
#
# The manual run is always launched from `main` (so the workflow definition and
# `breeze` come from `main`), and the `ref` input selects the commit-ish
# (branch, tag or commit hash) whose sources the constraints are refreshed from.
# The `constraints-X-Y` branch to push to is derived from that ref's
# `dev/breeze/src/airflow_breeze/branch_defaults.py`, so no cherry-pick to the
# `vX-Y-test` / `vX-Y-stable` branch is needed.
# See dev/MANUALLY_GENERATING_IMAGE_CACHE_AND_CONSTRAINTS.md for details.
on: # yamllint disable-line rule:truthy
push:
branches:
- main
- v[0-9]+-[0-9]+-test
paths:
- 'uv.lock'
workflow_dispatch:
inputs:
ref:
description: >-
Commit-ish to refresh constraints from (branch, tag or commit hash),
e.g. `v3-3-test`, `v3-3-stable`, `constraints-3-3` or a tag/hash.
The matching `constraints-X-Y` branch is derived from that ref.
required: true
type: string
upgrade-to-newer-dependencies:
description: >-
Re-resolve to the newest matching dependencies (picks up newly
released providers/dependencies from PyPI). Leave enabled when
refreshing constraints before promoting an RC.
type: boolean
default: true
permissions:
contents: read

concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
group: ${{ github.workflow }}-${{ github.ref }}-${{ inputs.ref }}
cancel-in-progress: true
env:
GITHUB_REPOSITORY: ${{ github.repository }}
Expand All @@ -39,6 +68,25 @@ jobs:
build-info:
name: "Build info"
runs-on: ["ubuntu-22.04"]
# Automatic uv.lock-push runs are unrestricted; manual (workflow_dispatch)
# runs are limited to release managers (same allowlist as the prod image
# release workflow) because they push to the protected constraints-* branches.
if: >-
github.event_name != 'workflow_dispatch' ||
contains(fromJSON('[
"ashb",
"bugraoz93",
"eladkal",
"ephraimbuddy",
"jedcunningham",
"jscheffl",
"kaxil",
"pierrejeambrun",
"potiuk",
"utkarsharma2",
"vincbeck",
"vatsrahul1001",
]'), github.event.sender.login)
outputs:
default-branch: ${{ steps.selective-checks.outputs.default-branch }}
default-constraints-branch: ${{ steps.selective-checks.outputs.default-constraints-branch }}
Expand All @@ -49,14 +97,10 @@ jobs:
- name: "Cleanup repo"
shell: bash
run: sudo rm -rf ${GITHUB_WORKSPACE}/*
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
persist-credentials: false
- name: Fetch incoming commit ${{ github.sha }} with its parent
- name: "Checkout ${{ inputs.ref || github.ref }} ( ${{ inputs.ref || github.sha }} )"
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
ref: ${{ github.sha }}
ref: ${{ inputs.ref || github.sha }}
fetch-depth: 2
persist-credentials: false
- name: "Install Breeze"
Expand All @@ -75,7 +119,7 @@ jobs:
id: selective-checks
env:
PR_LABELS: "[]"
COMMIT_REF: "${{ github.sha }}"
COMMIT_REF: "${{ inputs.ref || github.sha }}"
VERBOSE: "false"
GITHUB_CONTEXT_INPUT: "${{ runner.temp }}/github_context.json"
run: breeze ci selective-check 2>> ${GITHUB_OUTPUT}
Expand All @@ -96,8 +140,11 @@ jobs:
python-versions: ${{ needs.build-info.outputs.python-versions }}
branch: ${{ needs.build-info.outputs.default-branch }}
constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }}
checkout-ref: ${{ inputs.ref }}
use-uv: "true"
upgrade-to-newer-dependencies: "false"
upgrade-to-newer-dependencies: >-
${{ github.event_name == 'workflow_dispatch'
&& inputs.upgrade-to-newer-dependencies && 'true' || 'false' }}
docker-cache: "registry"
disable-airflow-repo-cache: "false"

Expand All @@ -113,6 +160,7 @@ jobs:
generate-pypi-constraints: "true"
generate-no-providers-constraints: "true"
debug-resources: "false"
checkout-ref: ${{ inputs.ref }}
use-uv: "true"

update-constraints:
Expand All @@ -125,6 +173,7 @@ jobs:
packages: read
env:
PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }}
CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }}
steps:
- name: "Cleanup repo"
shell: bash
Expand All @@ -133,14 +182,11 @@ jobs:
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
persist-credentials: false
- name: "Set constraints branch name"
id: constraints-branch
run: ./scripts/ci/constraints/ci_branch_constraints.sh >> ${GITHUB_OUTPUT}
- name: Checkout ${{ steps.constraints-branch.outputs.branch }}
- name: Checkout ${{ env.CONSTRAINTS_BRANCH }}
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
path: "constraints"
ref: ${{ steps.constraints-branch.outputs.branch }}
ref: ${{ env.CONSTRAINTS_BRANCH }}
persist-credentials: true
fetch-depth: 0
- name: "Download constraints from the generate-constraints job"
Expand Down
49 changes: 49 additions & 0 deletions dev/MANUALLY_GENERATING_IMAGE_CACHE_AND_CONSTRAINTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
- [What the command does](#what-the-command-does)
- [Manually generating constraint files](#manually-generating-constraint-files)
- [Why we need to generate constraint files manually](#why-we-need-to-generate-constraint-files-manually)
- [How to refresh constraints via the CI workflow (recommended)](#how-to-refresh-constraints-via-the-ci-workflow-recommended)
- [How to generate constraint files](#how-to-generate-constraint-files)
- [Is it safe to generate constraints manually?](#is-it-safe-to-generate-constraints-manually)
- [Manually updating already tagged constraint files](#manually-updating-already-tagged-constraint-files)
Expand Down Expand Up @@ -213,6 +214,54 @@ with tests, but we KNOW that the tip of the branch is good and we want to releas
we want to move the PRs of contributors to start using the new constraints. This should be done with caution
and you need to be sure what you are doing, but you can always do it manually if you want.

## How to refresh constraints via the CI workflow (recommended)

The easiest way to refresh the constraints - for example to pick up newly released
providers/dependencies from PyPI just before promoting an RC - is to trigger the
[`Update constraints`](../.github/workflows/update-constraints-on-push.yml) workflow manually
instead of running the `breeze` commands locally. The workflow runs exactly the same steps
that run automatically when `uv.lock` changes, builds the CI images, generates all constraint
flavours and commits/pushes them to the matching `constraints-*` branch.

The manual run is always launched from `main` and takes a `ref` input that selects the
commit-ish (branch, tag or commit hash) whose sources the constraints are refreshed from. You
do **not** select the release branch in GitHub's branch dropdown, and you do **not** need to
cherry-pick anything to the `vX-Y-test` / `vX-Y-stable` branch first - the workflow definition
and `breeze` come from `main`, while the sources come from the `ref` you pass. The only
requirement is that the `breeze` constraint-generation commands work against that `ref` (they
are stable across recent branches).

Manual runs are restricted to release managers (the same allowlist as the prod image release
workflow); the automatic `uv.lock`-push runs are not restricted.

To run it:

1. Go to the [`Update constraints`](https://github.com/apache/airflow/actions/workflows/update-constraints-on-push.yml)
workflow in the Actions tab.
2. Click **Run workflow** and keep the branch set to `main` (this is where the workflow runs
from - it is not the branch whose constraints get refreshed).
3. In the **ref** field, enter the commit-ish to refresh constraints from - for example
`v3-3-test`, `v3-3-stable`, `constraints-3-3`, an RC tag, or a commit hash. The matching
`constraints-X-Y` branch to push to is derived automatically from that ref's
`dev/breeze/src/airflow_breeze/branch_defaults.py`, so pointing at anything on the 3.3 line
refreshes `constraints-3-3`.
4. Keep **Re-resolve to the newest matching dependencies** enabled (the default) so the run
picks up the latest released providers/dependencies from PyPI. Disable it only if you want
to regenerate constraints strictly from that ref's `uv.lock` without upgrading.
5. Once the run finishes, verify the new commit on the matching constraints branch
(for example `constraints-3-3` for a 3.3 refresh):

https://github.com/apache/airflow/commits/constraints-<major>-<minor>/

> [!NOTE]
> Because `v3-3-test` and `v3-3-stable` can diverge while RCs are being voted on (fixes are
> cherry-picked to `v3-3-test`), be deliberate about which `ref` you refresh from. Refresh from
> the ref that matches the artifacts you are about to promote - typically `v3-3-stable` (or the
> RC tag) for a release, not `v3-3-test`.

If you cannot or do not want to use the workflow, you can still generate the constraints
locally with the `breeze` commands below.

## How to generate constraint files

```bash
Expand Down
10 changes: 10 additions & 0 deletions dev/README_RELEASE_AIRFLOW.md
Original file line number Diff line number Diff line change
Expand Up @@ -1372,6 +1372,16 @@ breeze release-management start-release \

Note: The `--task-sdk-version` parameter is optional. If you are releasing Airflow without a corresponding Task SDK release, you can omit this parameter.

Note: When it reaches the constraints step, `start-release` asks whether to base the final
`constraints-${VERSION}` tag on the latest `constraints-X-Y` branch tip instead of the RC
constraints tag. The RC constraints are frozen when the RC is cut, so if any providers were
released (or constraints were otherwise refreshed - see
[MANUALLY_GENERATING_IMAGE_CACHE_AND_CONSTRAINTS.md](MANUALLY_GENERATING_IMAGE_CACHE_AND_CONSTRAINTS.md))
after the last RC and you want the released constraints to reflect that, answer **yes** to tag the
`constraints-X-Y` branch tip. Otherwise (the default) the final tag matches the RC exactly. If you
do refresh, run the `Update constraints` workflow from `main` with `ref` set to the ref you are
releasing (typically `v3-*-stable`) **before** running `start-release`.


4. Make sure to update Airflow version in ``v3-*-test`` branch after cherry-picking to X.Y.1 in
``airflow/__init__.py``
Expand Down
21 changes: 19 additions & 2 deletions dev/breeze/src/airflow_breeze/commands/release_command.py
Original file line number Diff line number Diff line change
Expand Up @@ -343,10 +343,27 @@ def upload_to_pypi(version, task_sdk_version=None):
)


def get_constraints_branch_for_version(version: str) -> str:
major, minor = version.split(".")[:2]
return f"constraints-{major}-{minor}"


def retag_constraints(release_candidate, version):
if confirm_action(f"Retag constraints for {release_candidate} as {version}?"):
# By default the final ``constraints-<version>`` tag is created from the RC constraints tag.
# If the constraints were refreshed after the last RC (e.g. to pick up newly released
# providers - see dev/MANUALLY_GENERATING_IMAGE_CACHE_AND_CONSTRAINTS.md) the tip of the
# ``constraints-X-Y`` branch is newer than the RC tag and should be tagged instead.
constraints_branch = get_constraints_branch_for_version(version)
source_ref = f"constraints-{release_candidate}"
if confirm_action(
f"Base the final constraints on the latest '{constraints_branch}' branch tip instead of the "
f"'{source_ref}' tag? Choose yes if you refreshed constraints after {release_candidate}."
):
run_command(["git", "fetch", "origin", constraints_branch], check=True)
source_ref = f"origin/{constraints_branch}"
if confirm_action(f"Retag constraints from {source_ref} as {version}?"):
run_command(
["git", "checkout", f"constraints-{release_candidate}"],
["git", "checkout", source_ref],
check=True,
)
run_command(
Expand Down
52 changes: 52 additions & 0 deletions dev/breeze/tests/test_release_command.py
Original file line number Diff line number Diff line change
Expand Up @@ -538,3 +538,55 @@ def fake_getcwd() -> str:
assert "task-sdk" not in " ".join(console_messages).lower()
assert run_command_calls == []
assert chdir_calls == [svn_release_repo, "/original/dir"]


@pytest.mark.parametrize(
("version", "expected_branch"),
[
("3.3.0", "constraints-3-3"),
("3.0.5", "constraints-3-0"),
("3.12.10", "constraints-3-12"),
],
)
def test_get_constraints_branch_for_version(release_cmd, version, expected_branch):
assert release_cmd.get_constraints_branch_for_version(version) == expected_branch


def test_retag_constraints_from_rc_tag_by_default(monkeypatch, release_cmd):
run_command_calls: list[list[str]] = []

def fake_confirm_action(prompt: str, **_kwargs) -> bool:
# Decline basing on the branch tip; accept the retag and the push.
return not prompt.startswith("Base the final constraints on the latest")

monkeypatch.setattr(release_cmd, "confirm_action", fake_confirm_action)
monkeypatch.setattr(release_cmd, "run_command", lambda cmd, **_kwargs: run_command_calls.append(cmd))

release_cmd.retag_constraints("3.3.0rc2", "3.3.0")

assert ["git", "checkout", "constraints-3.3.0rc2"] in run_command_calls
assert ["git", "tag", "-s", "constraints-3.3.0", "-m", "Constraints for Apache Airflow 3.3.0"] in (
run_command_calls
)
assert ["git", "push", "origin", "tag", "constraints-3.3.0"] in run_command_calls
# The branch tip is not fetched when basing on the RC tag.
assert not any(cmd[:2] == ["git", "fetch"] for cmd in run_command_calls)


def test_retag_constraints_from_branch_tip_when_confirmed(monkeypatch, release_cmd):
run_command_calls: list[list[str]] = []

def fake_confirm_action(prompt: str, **_kwargs) -> bool:
return True

monkeypatch.setattr(release_cmd, "confirm_action", fake_confirm_action)
monkeypatch.setattr(release_cmd, "run_command", lambda cmd, **_kwargs: run_command_calls.append(cmd))

release_cmd.retag_constraints("3.3.0rc2", "3.3.0")

assert ["git", "fetch", "origin", "constraints-3-3"] in run_command_calls
assert ["git", "checkout", "origin/constraints-3-3"] in run_command_calls
assert ["git", "checkout", "constraints-3.3.0rc2"] not in run_command_calls
assert ["git", "tag", "-s", "constraints-3.3.0", "-m", "Constraints for Apache Airflow 3.3.0"] in (
run_command_calls
)
Loading