Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion .github/workflows/generate-constraints.yml
Original file line number Diff line number Diff line change
Expand Up @@ -135,7 +135,9 @@ jobs:
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: constraints-${{ matrix.python-version }}
path: ./files/constraints-${{ matrix.python-version }}/constraints-*.txt
path: |
./files/constraints-${{ matrix.python-version }}/constraints-*.txt
./files/constraints-${{ matrix.python-version }}/build-constraints-*.txt
retention-days: 7
if-no-files-found: error
- name: "Dependency upgrade summary"
Expand Down
102 changes: 100 additions & 2 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -870,6 +870,78 @@ function common::get_constraints_location() {
fi
}

function common::get_build_constraints_location() {
# Explicit override via env var - fail if set but file/URL is missing
if [[ -n ${AIRFLOW_BUILD_CONSTRAINTS_LOCATION=} ]]; then
if [[ ${AIRFLOW_BUILD_CONSTRAINTS_LOCATION} =~ http.* ]]; then
echo
echo "${COLOR_BLUE}Downloading build constraints from ${AIRFLOW_BUILD_CONSTRAINTS_LOCATION} to ${HOME}/build-constraints.txt${COLOR_RESET}"
echo
if ! curl -sSf -o "${HOME}/build-constraints.txt" "${AIRFLOW_BUILD_CONSTRAINTS_LOCATION}"; then
echo
echo "${COLOR_RED}Build constraints file not found at explicitly set ${AIRFLOW_BUILD_CONSTRAINTS_LOCATION}${COLOR_RESET}"
echo
exit 1
fi
else
if [[ ! -f "${AIRFLOW_BUILD_CONSTRAINTS_LOCATION}" ]]; then
echo
echo "${COLOR_RED}Build constraints file not found at explicitly set ${AIRFLOW_BUILD_CONSTRAINTS_LOCATION}${COLOR_RESET}"
echo
exit 1
fi
echo
echo "${COLOR_BLUE}Copying build constraints from ${AIRFLOW_BUILD_CONSTRAINTS_LOCATION} to ${HOME}/build-constraints.txt${COLOR_RESET}"
echo
cp "${AIRFLOW_BUILD_CONSTRAINTS_LOCATION}" "${HOME}/build-constraints.txt"
fi
return
fi

# Auto-detect constraints reference (idempotent - may already be set by get_constraints_location)
if [[ -z "${AIRFLOW_CONSTRAINTS_REFERENCE=}" ]]; then
if [[ ${AIRFLOW_VERSION} =~ v?2.* || ${AIRFLOW_VERSION} =~ v?3.* ]]; then
AIRFLOW_CONSTRAINTS_REFERENCE=constraints-${AIRFLOW_VERSION}
else
AIRFLOW_CONSTRAINTS_REFERENCE=${DEFAULT_CONSTRAINTS_BRANCH}
fi
fi

local constraints_base="https://raw.githubusercontent.com/${CONSTRAINTS_GITHUB_REPOSITORY}/${AIRFLOW_CONSTRAINTS_REFERENCE}"
local python_version
python_version=$(python -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")')
local build_constraints_url="${constraints_base}/build-constraints-${python_version}.txt"

echo
echo "${COLOR_BLUE}Downloading build constraints from ${build_constraints_url} to ${HOME}/build-constraints.txt${COLOR_RESET}"
echo
if ! curl -sSf -o "${HOME}/build-constraints.txt" "${build_constraints_url}"; then
echo
echo "${COLOR_YELLOW}Build constraints file not found at ${build_constraints_url} (not yet published for this version?).${COLOR_RESET}"
echo "${COLOR_YELLOW}Continuing without build constraints.${COLOR_RESET}"
echo
# Create/truncate to empty so the install flags function returns nothing
: > "${HOME}/build-constraints.txt"
fi
}

function common::get_build_constraints_install_flags() {
local build_constraints_file="${HOME}/build-constraints.txt"
if [[ ! -f "${build_constraints_file}" ]]; then
# No build constraints file - return empty (no flags)
return
fi
# Skip if file is empty (fallback from old constraints branch without build constraints)
if [[ ! -s "${build_constraints_file}" ]]; then
return
fi
if [[ ${PACKAGING_TOOL} == "uv" ]]; then
echo "--build-constraints ${build_constraints_file}"
else
echo "--build-constraint ${build_constraints_file}"
fi
}

function common::show_packaging_tool_version_and_location() {
echo "PATH=${PATH}"
echo "Installed pip: $(pip --version): $(which pip)"
Expand Down Expand Up @@ -1059,6 +1131,8 @@ function install_airflow_and_providers_from_docker_context_files(){
python_version=$(python -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")')
local local_constraints_file=/docker-context-files/constraints-"${python_version}"/${AIRFLOW_CONSTRAINTS_MODE}-"${python_version}".txt

local local_build_constraints_file=/docker-context-files/constraints-"${python_version}"/build-constraints-"${python_version}".txt

if [[ -f "${local_constraints_file}" ]]; then
echo
echo "${COLOR_BLUE}Installing docker-context-files distributions with constraints found in ${local_constraints_file}${COLOR_RESET}"
Expand All @@ -1069,6 +1143,13 @@ function install_airflow_and_providers_from_docker_context_files(){
echo "${COLOR_BLUE}Copying ${local_constraints_file} to ${HOME}/constraints.txt${COLOR_RESET}"
echo
cp "${local_constraints_file}" "${HOME}/constraints.txt"
# Override build constraints with local docker-context file if available
if [[ -f "${local_build_constraints_file}" ]]; then
echo
echo "${COLOR_BLUE}Copying ${local_build_constraints_file} to ${HOME}/build-constraints.txt${COLOR_RESET}"
echo
cp "${local_build_constraints_file}" "${HOME}/build-constraints.txt"
fi
else
echo
echo "${COLOR_BLUE}Installing docker-context-files distributions with constraints from GitHub${COLOR_RESET}"
Expand All @@ -1083,8 +1164,10 @@ function install_airflow_and_providers_from_docker_context_files(){
fi

set -x
# shellcheck disable=SC2046
if ! ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} \
${ADDITIONAL_PIP_INSTALL_FLAGS} \
$(common::get_build_constraints_install_flags) \
"${flags[@]}" \
"${install_airflow_distribution[@]}" "${install_airflow_core_distribution[@]}" "${airflow_distributions[@]}"; then
set +x
Expand Down Expand Up @@ -1121,7 +1204,9 @@ function install_all_other_distributions_from_docker_context_files() {
grep -v apache_airflow | grep -v apache-airflow || true)
if [[ -n "${reinstalling_other_distributions}" ]]; then
set -x
# shellcheck disable=SC2046
${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${ADDITIONAL_PIP_INSTALL_FLAGS} \
$(common::get_build_constraints_install_flags) \
--force-reinstall --no-deps --no-index ${reinstalling_other_distributions}
common::install_packaging_tools
set +x
Expand All @@ -1132,6 +1217,7 @@ common::get_colors
common::get_packaging_tool
common::get_airflow_version_specification
common::get_constraints_location
common::get_build_constraints_location
common::show_packaging_tool_version_and_location

install_airflow_and_providers_from_docker_context_files
Expand Down Expand Up @@ -1266,6 +1352,7 @@ function install_from_sources() {
}

function install_from_external_spec() {
common::get_build_constraints_location
local installation_command_flags
if [[ ${AIRFLOW_INSTALLATION_METHOD} == "apache-airflow" ]]; then
installation_command_flags="apache-airflow[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}"
Expand All @@ -1288,14 +1375,16 @@ function install_from_external_spec() {
echo "${COLOR_BLUE}Installing all packages with highest resolutions. Installation method: ${AIRFLOW_INSTALLATION_METHOD}${COLOR_RESET}"
echo
set -x
${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${UPGRADE_TO_HIGHEST_RESOLUTION} ${ADDITIONAL_PIP_INSTALL_FLAGS} ${installation_command_flags}
# shellcheck disable=SC2046
${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${UPGRADE_TO_HIGHEST_RESOLUTION} ${ADDITIONAL_PIP_INSTALL_FLAGS} $(common::get_build_constraints_install_flags) ${installation_command_flags}
set +x
else
echo
echo "${COLOR_BLUE}Installing all packages with constraints. Installation method: ${AIRFLOW_INSTALLATION_METHOD}${COLOR_RESET}"
echo
set -x
if ! ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${ADDITIONAL_PIP_INSTALL_FLAGS} ${installation_command_flags} --constraint "${HOME}/constraints.txt"; then
# shellcheck disable=SC2046
if ! ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${ADDITIONAL_PIP_INSTALL_FLAGS} $(common::get_build_constraints_install_flags) ${installation_command_flags} --constraint "${HOME}/constraints.txt"; then
set +x
if [[ ${AIRFLOW_FALLBACK_NO_CONSTRAINTS_INSTALLATION} != "true" ]]; then
echo
Expand Down Expand Up @@ -1329,6 +1418,9 @@ function install_airflow_when_building_images() {
fi
# Determine the installation_command_flags based on AIRFLOW_INSTALLATION_METHOD method
if [[ ${AIRFLOW_INSTALLATION_METHOD} == "." ]]; then
# Source installs use uv sync, which does not support --build-constraints.
# Remove stale build constraints from a previous non-source install run.
rm -f "${HOME}/build-constraints.txt"
install_from_sources
else
install_from_external_spec
Expand Down Expand Up @@ -1366,8 +1458,10 @@ function install_additional_dependencies() {
echo "${COLOR_BLUE}Installing additional dependencies while upgrading to newer dependencies${COLOR_RESET}"
echo
set -x
# shellcheck disable=SC2046
${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${UPGRADE_TO_HIGHEST_RESOLUTION} \
${ADDITIONAL_PIP_INSTALL_FLAGS} \
$(common::get_build_constraints_install_flags) \
${ADDITIONAL_PYTHON_DEPS}
set +x
common::install_packaging_tools
Expand All @@ -1381,8 +1475,10 @@ function install_additional_dependencies() {
echo "${COLOR_BLUE}Installing additional dependencies upgrading only if needed${COLOR_RESET}"
echo
set -x
# shellcheck disable=SC2046
${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${UPGRADE_IF_NEEDED} \
${ADDITIONAL_PIP_INSTALL_FLAGS} \
$(common::get_build_constraints_install_flags) \
${ADDITIONAL_PYTHON_DEPS}
set +x
common::install_packaging_tools
Expand All @@ -1398,6 +1494,7 @@ common::get_colors
common::get_packaging_tool
common::get_airflow_version_specification
common::get_constraints_location
common::get_build_constraints_location
common::show_packaging_tool_version_and_location

install_additional_dependencies
Expand Down Expand Up @@ -1913,6 +2010,7 @@ ARG CONSTRAINTS_GITHUB_REPOSITORY="apache/airflow"
ARG AIRFLOW_CONSTRAINTS_MODE="constraints"
ARG AIRFLOW_CONSTRAINTS_REFERENCE=""
ARG AIRFLOW_CONSTRAINTS_LOCATION=""
ARG AIRFLOW_BUILD_CONSTRAINTS_LOCATION=""
ARG DEFAULT_CONSTRAINTS_BRANCH="constraints-main"
# By default do not fallback to installation without constraints because it can hide problems with constraints
ARG AIRFLOW_FALLBACK_NO_CONSTRAINTS_INSTALLATION="false"
Expand Down
96 changes: 93 additions & 3 deletions Dockerfile.ci
Original file line number Diff line number Diff line change
Expand Up @@ -810,6 +810,78 @@ function common::get_constraints_location() {
fi
}

function common::get_build_constraints_location() {
# Explicit override via env var - fail if set but file/URL is missing
if [[ -n ${AIRFLOW_BUILD_CONSTRAINTS_LOCATION=} ]]; then
if [[ ${AIRFLOW_BUILD_CONSTRAINTS_LOCATION} =~ http.* ]]; then
echo
echo "${COLOR_BLUE}Downloading build constraints from ${AIRFLOW_BUILD_CONSTRAINTS_LOCATION} to ${HOME}/build-constraints.txt${COLOR_RESET}"
echo
if ! curl -sSf -o "${HOME}/build-constraints.txt" "${AIRFLOW_BUILD_CONSTRAINTS_LOCATION}"; then
echo
echo "${COLOR_RED}Build constraints file not found at explicitly set ${AIRFLOW_BUILD_CONSTRAINTS_LOCATION}${COLOR_RESET}"
echo
exit 1
fi
else
if [[ ! -f "${AIRFLOW_BUILD_CONSTRAINTS_LOCATION}" ]]; then
echo
echo "${COLOR_RED}Build constraints file not found at explicitly set ${AIRFLOW_BUILD_CONSTRAINTS_LOCATION}${COLOR_RESET}"
echo
exit 1
fi
echo
echo "${COLOR_BLUE}Copying build constraints from ${AIRFLOW_BUILD_CONSTRAINTS_LOCATION} to ${HOME}/build-constraints.txt${COLOR_RESET}"
echo
cp "${AIRFLOW_BUILD_CONSTRAINTS_LOCATION}" "${HOME}/build-constraints.txt"
fi
return
fi

# Auto-detect constraints reference (idempotent - may already be set by get_constraints_location)
if [[ -z "${AIRFLOW_CONSTRAINTS_REFERENCE=}" ]]; then
if [[ ${AIRFLOW_VERSION} =~ v?2.* || ${AIRFLOW_VERSION} =~ v?3.* ]]; then
AIRFLOW_CONSTRAINTS_REFERENCE=constraints-${AIRFLOW_VERSION}
else
AIRFLOW_CONSTRAINTS_REFERENCE=${DEFAULT_CONSTRAINTS_BRANCH}
fi
fi

local constraints_base="https://raw.githubusercontent.com/${CONSTRAINTS_GITHUB_REPOSITORY}/${AIRFLOW_CONSTRAINTS_REFERENCE}"
local python_version
python_version=$(python -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")')
local build_constraints_url="${constraints_base}/build-constraints-${python_version}.txt"

echo
echo "${COLOR_BLUE}Downloading build constraints from ${build_constraints_url} to ${HOME}/build-constraints.txt${COLOR_RESET}"
echo
if ! curl -sSf -o "${HOME}/build-constraints.txt" "${build_constraints_url}"; then
echo
echo "${COLOR_YELLOW}Build constraints file not found at ${build_constraints_url} (not yet published for this version?).${COLOR_RESET}"
echo "${COLOR_YELLOW}Continuing without build constraints.${COLOR_RESET}"
echo
# Create/truncate to empty so the install flags function returns nothing
: > "${HOME}/build-constraints.txt"
fi
}

function common::get_build_constraints_install_flags() {
local build_constraints_file="${HOME}/build-constraints.txt"
if [[ ! -f "${build_constraints_file}" ]]; then
# No build constraints file - return empty (no flags)
return
fi
# Skip if file is empty (fallback from old constraints branch without build constraints)
if [[ ! -s "${build_constraints_file}" ]]; then
return
fi
if [[ ${PACKAGING_TOOL} == "uv" ]]; then
echo "--build-constraints ${build_constraints_file}"
else
echo "--build-constraint ${build_constraints_file}"
fi
}

function common::show_packaging_tool_version_and_location() {
echo "PATH=${PATH}"
echo "Installed pip: $(pip --version): $(which pip)"
Expand Down Expand Up @@ -971,6 +1043,7 @@ function install_from_sources() {
}

function install_from_external_spec() {
common::get_build_constraints_location
local installation_command_flags
if [[ ${AIRFLOW_INSTALLATION_METHOD} == "apache-airflow" ]]; then
installation_command_flags="apache-airflow[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}"
Expand All @@ -993,14 +1066,16 @@ function install_from_external_spec() {
echo "${COLOR_BLUE}Installing all packages with highest resolutions. Installation method: ${AIRFLOW_INSTALLATION_METHOD}${COLOR_RESET}"
echo
set -x
${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${UPGRADE_TO_HIGHEST_RESOLUTION} ${ADDITIONAL_PIP_INSTALL_FLAGS} ${installation_command_flags}
# shellcheck disable=SC2046
${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${UPGRADE_TO_HIGHEST_RESOLUTION} ${ADDITIONAL_PIP_INSTALL_FLAGS} $(common::get_build_constraints_install_flags) ${installation_command_flags}
set +x
else
echo
echo "${COLOR_BLUE}Installing all packages with constraints. Installation method: ${AIRFLOW_INSTALLATION_METHOD}${COLOR_RESET}"
echo
set -x
if ! ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${ADDITIONAL_PIP_INSTALL_FLAGS} ${installation_command_flags} --constraint "${HOME}/constraints.txt"; then
# shellcheck disable=SC2046
if ! ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${ADDITIONAL_PIP_INSTALL_FLAGS} $(common::get_build_constraints_install_flags) ${installation_command_flags} --constraint "${HOME}/constraints.txt"; then
set +x
if [[ ${AIRFLOW_FALLBACK_NO_CONSTRAINTS_INSTALLATION} != "true" ]]; then
echo
Expand Down Expand Up @@ -1034,6 +1109,9 @@ function install_airflow_when_building_images() {
fi
# Determine the installation_command_flags based on AIRFLOW_INSTALLATION_METHOD method
if [[ ${AIRFLOW_INSTALLATION_METHOD} == "." ]]; then
# Source installs use uv sync, which does not support --build-constraints.
# Remove stale build constraints from a previous non-source install run.
rm -f "${HOME}/build-constraints.txt"
install_from_sources
else
install_from_external_spec
Expand Down Expand Up @@ -1071,8 +1149,10 @@ function install_additional_dependencies() {
echo "${COLOR_BLUE}Installing additional dependencies while upgrading to newer dependencies${COLOR_RESET}"
echo
set -x
# shellcheck disable=SC2046
${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${UPGRADE_TO_HIGHEST_RESOLUTION} \
${ADDITIONAL_PIP_INSTALL_FLAGS} \
$(common::get_build_constraints_install_flags) \
${ADDITIONAL_PYTHON_DEPS}
set +x
common::install_packaging_tools
Expand All @@ -1086,8 +1166,10 @@ function install_additional_dependencies() {
echo "${COLOR_BLUE}Installing additional dependencies upgrading only if needed${COLOR_RESET}"
echo
set -x
# shellcheck disable=SC2046
${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${UPGRADE_IF_NEEDED} \
${ADDITIONAL_PIP_INSTALL_FLAGS} \
$(common::get_build_constraints_install_flags) \
${ADDITIONAL_PYTHON_DEPS}
set +x
common::install_packaging_tools
Expand All @@ -1103,6 +1185,7 @@ common::get_colors
common::get_packaging_tool
common::get_airflow_version_specification
common::get_constraints_location
common::get_build_constraints_location
common::show_packaging_tool_version_and_location

install_additional_dependencies
Expand Down Expand Up @@ -1365,8 +1448,14 @@ function determine_airflow_to_use() {
local constraint_file="/tmp/constraints-from-lock.txt"
uv export --frozen --no-hashes --no-emit-project --no-emit-workspace --no-editable --no-header \
--no-annotate > "${constraint_file}" 2>/dev/null || true
local build_constraints_file="${AIRFLOW_SOURCES}/files/constraints-${PYTHON_MAJOR_MINOR_VERSION}/build-constraints-${PYTHON_MAJOR_MINOR_VERSION}.txt"
local build_constraints_args=""
if [[ -s "${build_constraints_file}" ]]; then
build_constraints_args="--build-constraints ${build_constraints_file}"
fi
# shellcheck disable=SC2086
uv run --no-cache /opt/airflow/scripts/in_container/install_development_dependencies.py \
--constraint "${constraint_file}"
--constraint "${constraint_file}" ${build_constraints_args}
# Some packages might leave legacy typing module which causes test issues
# shellcheck disable=SC2086
${PACKAGING_TOOL_CMD} uninstall ${EXTRA_UNINSTALL_FLAGS} typing || true
Expand Down Expand Up @@ -1756,6 +1845,7 @@ ARG CONSTRAINTS_GITHUB_REPOSITORY="apache/airflow"
ARG AIRFLOW_CONSTRAINTS_MODE="constraints-source-providers"
ARG AIRFLOW_CONSTRAINTS_REFERENCE=""
ARG AIRFLOW_CONSTRAINTS_LOCATION=""
ARG AIRFLOW_BUILD_CONSTRAINTS_LOCATION=""
ARG DEFAULT_CONSTRAINTS_BRANCH="constraints-main"
# By default fallback to installation without constraints because in CI image it should always be tried
ARG AIRFLOW_FALLBACK_NO_CONSTRAINTS_INSTALLATION="true"
Expand Down
Loading
Loading