From 5b5dd7f46247d3e2873d8fd0aa980bdf22b00707 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Sat, 21 May 2022 16:09:53 +0200 Subject: [PATCH] Better fix for constraint-reference (#23845) The previous fix (#23844) broke main on package verification as the package verification used the same parameter that was set to empty. This change rmeoves some remnant from the "bash" version where we had to check if variable was empty and also making the "constraint" parameters accepting default values from the current branch to be used also for build commands. (cherry picked from commit 10a0d8e7085f018b7328533030de76b48de747e2) --- .../commands/ci_image_commands.py | 9 ++------ .../commands/production_image_commands.py | 9 ++------ .../params/_common_build_params.py | 4 +--- .../airflow_breeze/params/build_ci_params.py | 5 +++++ .../params/build_prod_params.py | 21 ++++++++----------- .../airflow_breeze/utils/common_options.py | 9 ++++++++ .../utils/docker_command_utils.py | 5 +---- images/breeze/output-commands-hash.txt | 2 +- 8 files changed, 30 insertions(+), 34 deletions(-) diff --git a/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py b/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py index 2bbe49ab38bcf..1329ba607c4e4 100644 --- a/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py @@ -22,7 +22,6 @@ import click -from airflow_breeze.branch_defaults import DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH from airflow_breeze.commands.main_command import main from airflow_breeze.params.build_ci_params import BuildCiParams from airflow_breeze.params.shell_params import ShellParams @@ -36,6 +35,7 @@ option_additional_runtime_apt_deps, option_additional_runtime_apt_env, option_airflow_constraints_mode_ci, + option_airflow_constraints_reference_build, option_answer, option_build_multiple_images, option_debian_version, @@ -221,12 +221,7 @@ @option_runtime_apt_deps @option_force_build @option_airflow_constraints_mode_ci -@click.option( - "--airflow-constraints-reference", - default=DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH, - help="Constraint reference to use when building the image.", - envvar='AIRFLOW_CONSTRAINTS_REFERENCE', -) +@option_airflow_constraints_reference_build @option_tag_as_latest def build_image( verbose: bool, diff --git a/dev/breeze/src/airflow_breeze/commands/production_image_commands.py b/dev/breeze/src/airflow_breeze/commands/production_image_commands.py index 1f54d7b8dc203..32aed85d9ed14 100644 --- a/dev/breeze/src/airflow_breeze/commands/production_image_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/production_image_commands.py @@ -21,7 +21,6 @@ import click -from airflow_breeze.branch_defaults import DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH from airflow_breeze.commands.main_command import main from airflow_breeze.global_constants import ALLOWED_INSTALLATION_METHODS, DEFAULT_EXTRAS from airflow_breeze.params.build_prod_params import BuildProdParams @@ -35,6 +34,7 @@ option_additional_runtime_apt_deps, option_additional_runtime_apt_env, option_airflow_constraints_mode_prod, + option_airflow_constraints_reference_build, option_answer, option_build_multiple_images, option_debian_version, @@ -249,12 +249,7 @@ '--install-airflow-reference', help="Install Airflow using GitHub tag or branch.", ) -@click.option( - "--airflow-constraints-reference", - default=DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH, - help="Constraint reference to use when building the image.", - envvar='AIRFLOW_CONSTRAINTS_REFERENCE', -) +@option_airflow_constraints_reference_build @click.option('-V', '--install-airflow-version', help="Install version of Airflow from PyPI.") @option_additional_extras @option_additional_dev_apt_deps diff --git a/dev/breeze/src/airflow_breeze/params/_common_build_params.py b/dev/breeze/src/airflow_breeze/params/_common_build_params.py index 2fda00fbd532d..4f2139d20e723 100644 --- a/dev/breeze/src/airflow_breeze/params/_common_build_params.py +++ b/dev/breeze/src/airflow_breeze/params/_common_build_params.py @@ -21,7 +21,7 @@ from datetime import datetime from typing import List, Optional -from airflow_breeze.branch_defaults import AIRFLOW_BRANCH, DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH +from airflow_breeze.branch_defaults import AIRFLOW_BRANCH from airflow_breeze.utils.console import get_console from airflow_breeze.utils.platforms import get_real_platform @@ -42,12 +42,10 @@ class _CommonBuildParams: additional_runtime_apt_env: str = "" airflow_branch: str = AIRFLOW_BRANCH airflow_constraints_location: str = "" - airflow_constraints_reference: str = "constraints-main" answer: Optional[str] = None build_id: int = 0 constraints_github_repository: str = "apache/airflow" debian_version: str = "bullseye" - default_constraints_branch = DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH dev_apt_command: str = "" dev_apt_deps: str = "" docker_cache: str = "registry" diff --git a/dev/breeze/src/airflow_breeze/params/build_ci_params.py b/dev/breeze/src/airflow_breeze/params/build_ci_params.py index a22a96a34b0ea..b012993519a92 100644 --- a/dev/breeze/src/airflow_breeze/params/build_ci_params.py +++ b/dev/breeze/src/airflow_breeze/params/build_ci_params.py @@ -18,6 +18,7 @@ from pathlib import Path from typing import List +from airflow_breeze.branch_defaults import DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH from airflow_breeze.global_constants import get_airflow_version from airflow_breeze.params._common_build_params import _CommonBuildParams from airflow_breeze.utils.console import get_console @@ -31,6 +32,7 @@ class BuildCiParams(_CommonBuildParams): """ airflow_constraints_mode: str = "constraints-source-providers" + airflow_constraints_reference: str = DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH airflow_extras: str = "devel_ci" airflow_pre_cached_pip_packages: bool = True force_build: bool = False @@ -46,6 +48,9 @@ def image_type(self) -> str: @property def extra_docker_build_flags(self) -> List[str]: extra_ci_flags = [] + extra_ci_flags.extend( + ["--build-arg", f"AIRFLOW_CONSTRAINTS_REFERENCE={self.airflow_constraints_reference}"] + ) if self.airflow_constraints_location is not None and len(self.airflow_constraints_location) > 0: extra_ci_flags.extend( ["--build-arg", f"AIRFLOW_CONSTRAINTS_LOCATION={self.airflow_constraints_location}"] diff --git a/dev/breeze/src/airflow_breeze/params/build_prod_params.py b/dev/breeze/src/airflow_breeze/params/build_prod_params.py index 0cb6e84bd1b03..c480570f52ef4 100644 --- a/dev/breeze/src/airflow_breeze/params/build_prod_params.py +++ b/dev/breeze/src/airflow_breeze/params/build_prod_params.py @@ -41,6 +41,7 @@ class BuildProdParams(_CommonBuildParams): """ airflow_constraints_mode: str = "constraints" + airflow_constraints_reference: str = "" airflow_is_in_context: bool = False cleanup_context: bool = False disable_airflow_repo_cache: bool = False @@ -79,20 +80,16 @@ def args_for_remote_install(self) -> List: "AIRFLOW_SOURCES_TO=/empty", ] ) - if len(self.airflow_constraints_reference) > 0: + if re.match('v?2.*', self.airflow_version): build_args.extend( - ["--build-arg", f"AIRFLOW_CONSTRAINTS_REFERENCE={self.airflow_constraints_reference}"] + ["--build-arg", f"AIRFLOW_CONSTRAINTS_REFERENCE=constraints-{self.airflow_version}"] ) else: - if re.match('v?2.*', self.airflow_version): - build_args.extend( - ["--build-arg", f"AIRFLOW_CONSTRAINTS_REFERENCE=constraints-{self.airflow_version}"] - ) - else: - build_args.extend( - ["--build-arg", f"AIRFLOW_CONSTRAINTS_REFERENCE={self.default_constraints_branch}"] - ) - if len(self.airflow_constraints_location) > 0: + build_args.extend( + ["--build-arg", f"AIRFLOW_CONSTRAINTS_REFERENCE={self.airflow_constraints_reference}"] + ) + if self.airflow_constraints_location: + # override location if specified build_args.extend( ["--build-arg", f"AIRFLOW_CONSTRAINTS_LOCATION={self.airflow_constraints_location}"] ) @@ -165,7 +162,7 @@ def extra_docker_build_flags(self) -> List[str]: "--build-arg", f"AIRFLOW_INSTALLATION_METHOD={self.installation_method}", "--build-arg", - f"AIRFLOW_CONSTRAINTS_REFERENCE={self.default_constraints_branch}", + f"AIRFLOW_CONSTRAINTS_REFERENCE={self.airflow_constraints_reference}", ] ) diff --git a/dev/breeze/src/airflow_breeze/utils/common_options.py b/dev/breeze/src/airflow_breeze/utils/common_options.py index cfc85b51b1e53..545a6ec149912 100644 --- a/dev/breeze/src/airflow_breeze/utils/common_options.py +++ b/dev/breeze/src/airflow_breeze/utils/common_options.py @@ -19,6 +19,7 @@ import click +from airflow_breeze.branch_defaults import DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH from airflow_breeze.global_constants import ( ALLOWED_BACKENDS, ALLOWED_BUILD_CACHE, @@ -415,8 +416,16 @@ "--airflow-constraints-reference", help="Constraint reference to use. Useful with --use-airflow-version parameter to specify " "constraints for the installed version and to find newer dependencies", + default=DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH, envvar='AIRFLOW_CONSTRAINTS_REFERENCE', ) +option_airflow_constraints_reference_build = click.option( + "--airflow-constraints-reference", + default=DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH, + help="Constraint reference to use when building the image.", + envvar='AIRFLOW_CONSTRAINTS_REFERENCE', +) + option_airflow_constraints_mode_ci = click.option( '--airflow-constraints-mode', type=BetterChoice(ALLOWED_CONSTRAINTS_MODES_CI), diff --git a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py index 85361c8d2a910..d3d40bc6079f4 100644 --- a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py @@ -36,7 +36,7 @@ # We handle the ImportError so that autocomplete works with just click installed version = None # type: ignore[assignment] -from airflow_breeze.branch_defaults import AIRFLOW_BRANCH, DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH +from airflow_breeze.branch_defaults import AIRFLOW_BRANCH from airflow_breeze.global_constants import ( ALLOWED_PACKAGE_FORMATS, FLOWER_HOST_PORT, @@ -456,7 +456,6 @@ def update_expected_environment_variables(env: Dict[str, str]) -> None: :param env: environment variables to update with missing values if not set. """ set_value_to_default_if_not_set(env, 'AIRFLOW_CONSTRAINTS_MODE', "constraints-source-providers") - set_value_to_default_if_not_set(env, 'AIRFLOW_CONSTRAINTS_REFERENCE', DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH) set_value_to_default_if_not_set(env, 'AIRFLOW_EXTRAS', "") set_value_to_default_if_not_set(env, 'ANSWER', "") set_value_to_default_if_not_set(env, 'BREEZE', "true") @@ -470,7 +469,6 @@ def update_expected_environment_variables(env: Dict[str, str]) -> None: set_value_to_default_if_not_set(env, 'DB_RESET', "false") set_value_to_default_if_not_set(env, 'DEBIAN_VERSION', "bullseye") set_value_to_default_if_not_set(env, 'DEFAULT_BRANCH', AIRFLOW_BRANCH) - set_value_to_default_if_not_set(env, 'DEFAULT_CONSTRAINTS_BRANCH', DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH) set_value_to_default_if_not_set(env, 'ENABLED_SYSTEMS', "") set_value_to_default_if_not_set(env, 'ENABLE_TEST_COVERAGE', "false") set_value_to_default_if_not_set(env, 'GITHUB_REGISTRY_PULL_IMAGE_TAG', "latest") @@ -504,7 +502,6 @@ def update_expected_environment_variables(env: Dict[str, str]) -> None: "AIRFLOW_CI_IMAGE_WITH_TAG": "airflow_image_name_with_tag", "AIRFLOW_EXTRAS": "airflow_extras", "AIRFLOW_CONSTRAINTS_MODE": "airflow_constraints_mode", - "AIRFLOW_CONSTRAINTS_REFERENCE": "airflow_constraints_reference", "AIRFLOW_IMAGE_KUBERNETES": "airflow_image_kubernetes", "AIRFLOW_PROD_IMAGE": "airflow_image_name", "AIRFLOW_SOURCES": "airflow_sources", diff --git a/images/breeze/output-commands-hash.txt b/images/breeze/output-commands-hash.txt index c4279b56f9733..8520b2c3bd921 100644 --- a/images/breeze/output-commands-hash.txt +++ b/images/breeze/output-commands-hash.txt @@ -1 +1 @@ -dd085eb6d80d465b5299e4f258153e84 +43bc980dd30f21c6979990470870a936