diff --git a/Dockerfile b/Dockerfile index a58aa327347fb..42b9db931fcaa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -325,20 +325,12 @@ COPY <<"EOF" /install_pip_version.sh : "${AIRFLOW_PIP_VERSION:?Should be set}" -function install_pip_version() { - echo - echo "${COLOR_BLUE}Installing pip version ${AIRFLOW_PIP_VERSION}${COLOR_RESET}" - echo - pip install --disable-pip-version-check --no-cache-dir --upgrade "pip==${AIRFLOW_PIP_VERSION}" && - mkdir -p ${HOME}/.local/bin -} - common::get_colors common::get_airflow_version_specification common::override_pip_version_if_needed common::show_pip_version_and_location -install_pip_version +common::install_pip_version EOF # The content below is automatically copied from scripts/docker/install_airflow_dependencies_from_branch_tip.sh @@ -369,8 +361,7 @@ function install_airflow_dependencies_from_branch_tip() { ${ADDITIONAL_PIP_INSTALL_FLAGS} \ "https://github.com/${AIRFLOW_REPO}/archive/${AIRFLOW_BRANCH}.tar.gz#egg=apache-airflow[${AIRFLOW_EXTRAS}]" \ --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}" || true - # make sure correct PIP version is used - pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null + common::install_pip_version pip freeze | grep apache-airflow-providers | xargs pip uninstall --yes 2>/dev/null || true set +x echo @@ -445,6 +436,18 @@ function common::show_pip_version_and_location() { echo "pip on path: $(which pip)" echo "Using pip: $(pip --version)" } + +function common::install_pip_version() { + echo + echo "${COLOR_BLUE}Installing pip version ${AIRFLOW_PIP_VERSION}${COLOR_RESET}" + echo + if [[ ${AIRFLOW_PIP_VERSION} =~ .*https.* ]]; then + pip install --disable-pip-version-check --no-cache-dir "pip @ ${AIRFLOW_PIP_VERSION}" + else + pip install --disable-pip-version-check --no-cache-dir "pip==${AIRFLOW_PIP_VERSION}" + fi + mkdir -p "${HOME}/.local/bin" +} EOF # The content below is automatically copied from scripts/docker/pip @@ -531,8 +534,7 @@ function install_airflow_and_providers_from_docker_context_files(){ ${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS} set +x - # make sure correct PIP version is left installed - pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null + common::install_pip_version pip check } @@ -549,9 +551,8 @@ function install_all_other_packages_from_docker_context_files() { set -x pip install ${ADDITIONAL_PIP_INSTALL_FLAGS} \ --root-user-action ignore --force-reinstall --no-deps --no-index ${reinstalling_other_packages} - # make sure correct PIP version is used - pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null - set -x + common::install_pip_version + set +x fi } @@ -613,8 +614,7 @@ function install_airflow() { set +x fi - # make sure correct PIP version is used - pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null + common::install_pip_version echo echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}" echo @@ -628,15 +628,13 @@ function install_airflow() { ${ADDITIONAL_PIP_INSTALL_FLAGS} \ "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" \ --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}" - # make sure correct PIP version is used - pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null + common::install_pip_version # then upgrade if needed without using constraints to account for new limits in setup.py pip install --root-user-action ignore --upgrade --upgrade-strategy only-if-needed \ ${ADDITIONAL_PIP_INSTALL_FLAGS} \ ${AIRFLOW_INSTALL_EDITABLE_FLAG} \ "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" - # make sure correct PIP version is used - pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null + common::install_pip_version set +x echo echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}" @@ -675,8 +673,7 @@ function install_additional_dependencies() { pip install --root-user-action ignore --upgrade --upgrade-strategy eager \ ${ADDITIONAL_PIP_INSTALL_FLAGS} \ ${ADDITIONAL_PYTHON_DEPS} ${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS} - # make sure correct PIP version is used - pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null + common::install_pip_version set +x echo echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}" @@ -690,8 +687,7 @@ function install_additional_dependencies() { pip install --root-user-action ignore --upgrade --upgrade-strategy only-if-needed \ ${ADDITIONAL_PIP_INSTALL_FLAGS} \ ${ADDITIONAL_PYTHON_DEPS} - # make sure correct PIP version is used - pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null + common::install_pip_version set +x echo echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}" diff --git a/Dockerfile.ci b/Dockerfile.ci index f5ef0743cd4ba..38eadbdfd7998 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -285,20 +285,12 @@ COPY <<"EOF" /install_pip_version.sh : "${AIRFLOW_PIP_VERSION:?Should be set}" -function install_pip_version() { - echo - echo "${COLOR_BLUE}Installing pip version ${AIRFLOW_PIP_VERSION}${COLOR_RESET}" - echo - pip install --disable-pip-version-check --no-cache-dir --upgrade "pip==${AIRFLOW_PIP_VERSION}" && - mkdir -p ${HOME}/.local/bin -} - common::get_colors common::get_airflow_version_specification common::override_pip_version_if_needed common::show_pip_version_and_location -install_pip_version +common::install_pip_version EOF # The content below is automatically copied from scripts/docker/install_airflow_dependencies_from_branch_tip.sh @@ -329,8 +321,7 @@ function install_airflow_dependencies_from_branch_tip() { ${ADDITIONAL_PIP_INSTALL_FLAGS} \ "https://github.com/${AIRFLOW_REPO}/archive/${AIRFLOW_BRANCH}.tar.gz#egg=apache-airflow[${AIRFLOW_EXTRAS}]" \ --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}" || true - # make sure correct PIP version is used - pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null + common::install_pip_version pip freeze | grep apache-airflow-providers | xargs pip uninstall --yes 2>/dev/null || true set +x echo @@ -405,6 +396,18 @@ function common::show_pip_version_and_location() { echo "pip on path: $(which pip)" echo "Using pip: $(pip --version)" } + +function common::install_pip_version() { + echo + echo "${COLOR_BLUE}Installing pip version ${AIRFLOW_PIP_VERSION}${COLOR_RESET}" + echo + if [[ ${AIRFLOW_PIP_VERSION} =~ .*https.* ]]; then + pip install --disable-pip-version-check --no-cache-dir "pip @ ${AIRFLOW_PIP_VERSION}" + else + pip install --disable-pip-version-check --no-cache-dir "pip==${AIRFLOW_PIP_VERSION}" + fi + mkdir -p "${HOME}/.local/bin" +} EOF # The content below is automatically copied from scripts/docker/install_pipx_tools.sh @@ -480,8 +483,7 @@ function install_airflow() { set +x fi - # make sure correct PIP version is used - pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null + common::install_pip_version echo echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}" echo @@ -495,15 +497,13 @@ function install_airflow() { ${ADDITIONAL_PIP_INSTALL_FLAGS} \ "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" \ --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}" - # make sure correct PIP version is used - pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null + common::install_pip_version # then upgrade if needed without using constraints to account for new limits in setup.py pip install --root-user-action ignore --upgrade --upgrade-strategy only-if-needed \ ${ADDITIONAL_PIP_INSTALL_FLAGS} \ ${AIRFLOW_INSTALL_EDITABLE_FLAG} \ "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" - # make sure correct PIP version is used - pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null + common::install_pip_version set +x echo echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}" @@ -542,8 +542,7 @@ function install_additional_dependencies() { pip install --root-user-action ignore --upgrade --upgrade-strategy eager \ ${ADDITIONAL_PIP_INSTALL_FLAGS} \ ${ADDITIONAL_PYTHON_DEPS} ${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS} - # make sure correct PIP version is used - pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null + common::install_pip_version set +x echo echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}" @@ -557,8 +556,7 @@ function install_additional_dependencies() { pip install --root-user-action ignore --upgrade --upgrade-strategy only-if-needed \ ${ADDITIONAL_PIP_INSTALL_FLAGS} \ ${ADDITIONAL_PYTHON_DEPS} - # make sure correct PIP version is used - pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null + common::install_pip_version set +x echo echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}" diff --git a/scripts/docker/common.sh b/scripts/docker/common.sh index e7cfefaea79b9..fab02a3b4fb41 100644 --- a/scripts/docker/common.sh +++ b/scripts/docker/common.sh @@ -70,3 +70,15 @@ function common::show_pip_version_and_location() { echo "pip on path: $(which pip)" echo "Using pip: $(pip --version)" } + +function common::install_pip_version() { + echo + echo "${COLOR_BLUE}Installing pip version ${AIRFLOW_PIP_VERSION}${COLOR_RESET}" + echo + if [[ ${AIRFLOW_PIP_VERSION} =~ .*https.* ]]; then + pip install --disable-pip-version-check --no-cache-dir "pip @ ${AIRFLOW_PIP_VERSION}" + else + pip install --disable-pip-version-check --no-cache-dir "pip==${AIRFLOW_PIP_VERSION}" + fi + mkdir -p "${HOME}/.local/bin" +} diff --git a/scripts/docker/install_additional_dependencies.sh b/scripts/docker/install_additional_dependencies.sh index 9b42445d2b6cd..a745d576f363a 100644 --- a/scripts/docker/install_additional_dependencies.sh +++ b/scripts/docker/install_additional_dependencies.sh @@ -35,8 +35,7 @@ function install_additional_dependencies() { pip install --root-user-action ignore --upgrade --upgrade-strategy eager \ ${ADDITIONAL_PIP_INSTALL_FLAGS} \ ${ADDITIONAL_PYTHON_DEPS} ${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS} - # make sure correct PIP version is used - pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null + common::install_pip_version set +x echo echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}" @@ -50,8 +49,7 @@ function install_additional_dependencies() { pip install --root-user-action ignore --upgrade --upgrade-strategy only-if-needed \ ${ADDITIONAL_PIP_INSTALL_FLAGS} \ ${ADDITIONAL_PYTHON_DEPS} - # make sure correct PIP version is used - pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null + common::install_pip_version set +x echo echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}" diff --git a/scripts/docker/install_airflow.sh b/scripts/docker/install_airflow.sh index 3ee5b168aa64b..265d2089c3e60 100644 --- a/scripts/docker/install_airflow.sh +++ b/scripts/docker/install_airflow.sh @@ -69,8 +69,7 @@ function install_airflow() { set +x fi - # make sure correct PIP version is used - pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null + common::install_pip_version echo echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}" echo @@ -84,15 +83,13 @@ function install_airflow() { ${ADDITIONAL_PIP_INSTALL_FLAGS} \ "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" \ --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}" - # make sure correct PIP version is used - pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null + common::install_pip_version # then upgrade if needed without using constraints to account for new limits in setup.py pip install --root-user-action ignore --upgrade --upgrade-strategy only-if-needed \ ${ADDITIONAL_PIP_INSTALL_FLAGS} \ ${AIRFLOW_INSTALL_EDITABLE_FLAG} \ "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" - # make sure correct PIP version is used - pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null + common::install_pip_version set +x echo echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}" diff --git a/scripts/docker/install_airflow_dependencies_from_branch_tip.sh b/scripts/docker/install_airflow_dependencies_from_branch_tip.sh index 60f82efa492a9..2b58d03950e5b 100644 --- a/scripts/docker/install_airflow_dependencies_from_branch_tip.sh +++ b/scripts/docker/install_airflow_dependencies_from_branch_tip.sh @@ -52,8 +52,7 @@ function install_airflow_dependencies_from_branch_tip() { ${ADDITIONAL_PIP_INSTALL_FLAGS} \ "https://github.com/${AIRFLOW_REPO}/archive/${AIRFLOW_BRANCH}.tar.gz#egg=apache-airflow[${AIRFLOW_EXTRAS}]" \ --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}" || true - # make sure correct PIP version is used - pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null + common::install_pip_version pip freeze | grep apache-airflow-providers | xargs pip uninstall --yes 2>/dev/null || true set +x echo diff --git a/scripts/docker/install_from_docker_context_files.sh b/scripts/docker/install_from_docker_context_files.sh index e21814f40e9c2..250bbaed21a1d 100644 --- a/scripts/docker/install_from_docker_context_files.sh +++ b/scripts/docker/install_from_docker_context_files.sh @@ -85,8 +85,7 @@ function install_airflow_and_providers_from_docker_context_files(){ ${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS} set +x - # make sure correct PIP version is left installed - pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null + common::install_pip_version pip check } @@ -107,9 +106,8 @@ function install_all_other_packages_from_docker_context_files() { set -x pip install ${ADDITIONAL_PIP_INSTALL_FLAGS} \ --root-user-action ignore --force-reinstall --no-deps --no-index ${reinstalling_other_packages} - # make sure correct PIP version is used - pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null - set -x + common::install_pip_version + set +x fi } diff --git a/scripts/docker/install_pip_version.sh b/scripts/docker/install_pip_version.sh index 60d804cd10f09..efe4f23c2fb18 100644 --- a/scripts/docker/install_pip_version.sh +++ b/scripts/docker/install_pip_version.sh @@ -20,17 +20,9 @@ : "${AIRFLOW_PIP_VERSION:?Should be set}" -function install_pip_version() { - echo - echo "${COLOR_BLUE}Installing pip version ${AIRFLOW_PIP_VERSION}${COLOR_RESET}" - echo - pip install --disable-pip-version-check --no-cache-dir --upgrade "pip==${AIRFLOW_PIP_VERSION}" && - mkdir -p ${HOME}/.local/bin -} - common::get_colors common::get_airflow_version_specification common::override_pip_version_if_needed common::show_pip_version_and_location -install_pip_version +common::install_pip_version diff --git a/scripts/in_container/_in_container_utils.sh b/scripts/in_container/_in_container_utils.sh index 7d9221625fa4b..3240b2c90dacc 100644 --- a/scripts/in_container/_in_container_utils.sh +++ b/scripts/in_container/_in_container_utils.sh @@ -353,7 +353,12 @@ function setup_provider_packages() { function install_supported_pip_version() { - pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" + if [[ ${AIRFLOW_PIP_VERSION} =~ .*https.* ]]; then + pip install --disable-pip-version-check "pip @ ${AIRFLOW_PIP_VERSION}" + else + pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" + fi + } function filename_to_python_module() { diff --git a/scripts/in_container/run_prepare_airflow_packages.sh b/scripts/in_container/run_prepare_airflow_packages.sh index 0ba7871494194..68f9dc92be92a 100755 --- a/scripts/in_container/run_prepare_airflow_packages.sh +++ b/scripts/in_container/run_prepare_airflow_packages.sh @@ -34,7 +34,8 @@ function prepare_airflow_packages() { rm -rf -- *egg-info* rm -rf -- build - pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" "wheel==${WHEEL_VERSION}" + install_supported_pip_version + pip install "wheel==${WHEEL_VERSION}" local packages=()