Improve ansible host & venv bootstrap

Combined backport of: - https://review.openstack.org/484891 - https://review.openstack.org/485988 The current ansible bootstrap process tried to pin the versions of pip, setuptools and wheel on the host and also uses inherited python packages from the host. This causes problems when the host has a version of setuptools which is cannot be changed (perhaps due to some bug) or when the host has otherwise undesirable python packages. The ansible bootstrap process only needs to be concerned with whether pip is installed and understands how to use constraints. From there we can bootstrap the venv using get-pip and completely avoid package conflicts with the host. Once Ansible is bootstrapped, the pip_install role will ensure that pip, setuptools and wheel are correctly bootstrapped and pinned across all hosts. The pip_install role now uses constraints for the initial get-pip [1], so we can also remove the list of pip pins from group_vars. We remove pip, setuptools and wheel from the requirements.txt to allow the packages on the host to be installed in the bootstrap without forcing a change of pip/setuptools/wheel at the same time (which causes failures in some circumstances). [1] https://review.openstack.org/483905 Depends-On: https://review.openstack.org/558811 Change-Id: Ida84fb6bb726e1332f0e29ade51b67a5721f0785 (cherry picked from commit 5f7000e)
openstack · Apr 4, 2018 · 37ce304 · 37ce304
1 parent 1dc4e1b
commit 37ce304
Show file tree

Hide file tree

Showing 7 changed files with 47 additions and 62 deletions.
diff --git a/ansible-role-requirements.yml b/ansible-role-requirements.yml
@@ -9,7 +9,7 @@
 - name: pip_install
   scm: git
   src: https://git.openstack.org/openstack/openstack-ansible-pip_install
-  version: 351380e4e63707188d66e2699d5ca0ed3be4d0e3
+  version: 591683aaa9ff973d133e9c556229ff4ce327d675
 - name: galera_client
   scm: git
   src: https://git.openstack.org/openstack/openstack-ansible-galera_client

diff --git a/playbooks/inventory/group_vars/all.yml b/playbooks/inventory/group_vars/all.yml
@@ -49,13 +49,6 @@ repo_pkg_cache_port: 3142
 repo_pkg_cache_url: "http://{{ internal_lb_vip_address }}:{{ repo_pkg_cache_port }}"
 repo_release_path: "{{ openstack_repo_url }}/os-releases/{{ openstack_release }}/{{ os_distro_version }}"
 
-# These are pinned to ensure exactly the same behaviour forever!
-# These pins are updated through the sources-branch-updater script
-pip_packages:
-  - pip==9.0.1
-  - setuptools==38.5.0
-  - wheel==0.30.0
-
 pip_links:
   - { name: "openstack_release", link: "{{ repo_release_path }}/" }
 pip_lock_to_internal_repo: "{{ (pip_links | length) >= 1 }}"

diff --git a/requirements.txt b/requirements.txt
@@ -1,9 +1,6 @@
 # The order of packages is significant, because pip processes them in the order
 # of appearance. Changing the order has an impact on the overall integration
 # process, which may cause wedges in the gate later.
-pip>=7.1.0 # MIT
-setuptools!=24.0.0,!=34.0.0,!=34.0.1,!=34.0.2,!=34.0.3,!=34.1.0,!=34.1.1,!=34.2.0,!=34.3.0,!=34.3.1,!=34.3.2,!=36.2.0,>=16.0 # PSF/ZPL
-wheel # MIT
 pyasn1 # BSD
 pyOpenSSL>=0.14 # Apache-2.0
 ndg-httpsclient>=0.4.2;python_version<'3.0' # BSD

diff --git a/scripts/bootstrap-ansible.sh b/scripts/bootstrap-ansible.sh
@@ -33,10 +33,12 @@ export ANSIBLE_ROLE_FETCH_MODE=${ANSIBLE_ROLE_FETCH_MODE:-galaxy}
 # This script should be executed from the root directory of the cloned repo
 cd "$(dirname "${0}")/.."
 
+
 ## Functions -----------------------------------------------------------------
 info_block "Checking for required libraries." 2> /dev/null ||
     source scripts/scripts-library.sh
 
+
 ## Main ----------------------------------------------------------------------
 info_block "Bootstrapping System with Ansible"
 
@@ -73,9 +75,6 @@ case ${DISTRO_ID} in
         ;;
 esac
 
-# Install pip
-get_pip
-
 # Ensure we use the HTTPS/HTTP proxy with pip if it is specified
 PIP_OPTS=""
 if [ -n "$HTTPS_PROXY" ]; then
@@ -94,7 +93,17 @@ UPPER_CONSTRAINTS_PROTO=$([ "$PYTHON_VERSION" == $(echo -e "$PYTHON_VERSION\n2.7
 # Set the location of the constraints to use for all pip installations
 export UPPER_CONSTRAINTS_FILE=${UPPER_CONSTRAINTS_FILE:-"$UPPER_CONSTRAINTS_PROTO://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt?id=$(awk '/requirements_git_install_branch:/ {print $2}' playbooks/defaults/repo_packages/openstack_services.yml)"}
 
-# Make sure that host requirements are installed
+# Install pip on the host if it is not already installed,
+# but also make sure that it is at least version 9.x or above.
+PIP_VERSION=$(pip --version 2>/dev/null | awk '{print $2}' | cut -d. -f1)
+if [[ "${PIP_VERSION}" -lt "9" ]]; then
+  get_pip ${PYTHON_EXEC_PATH}
+  # Ensure that our shell knows about the new pip
+  hash -r pip
+fi
+
+# Install the requirements for the various python scripts
+# on to the host, including virtualenv.
 pip install ${PIP_OPTS} \
   --requirement requirements.txt \
   --constraint ${UPPER_CONSTRAINTS_FILE} \
@@ -106,24 +115,24 @@ pip install ${PIP_OPTS} \
 # Create a Virtualenv for the Ansible runtime
 virtualenv --python="${PYTHON_EXEC_PATH}" \
            --clear \
+           --no-pip --no-setuptools --no-wheel \
            /opt/ansible-runtime
 
+# Install pip, setuptools and wheel into the venv
+get_pip /opt/ansible-runtime/bin/python
+
 # The vars used to prepare the Ansible runtime venv
-PIP_OPTS+=" --upgrade"
 PIP_COMMAND="/opt/ansible-runtime/bin/pip"
+PIP_OPTS+=" --constraint global-requirement-pins.txt"
+PIP_OPTS+=" --constraint ${UPPER_CONSTRAINTS_FILE}"
 
 # When upgrading there will already be a pip.conf file locking pip down to the
 # repo server, in such cases it may be necessary to use --isolated because the
 # repo server does not meet the specified requirements.
 
-# Ensure we are running the required versions of pip, wheel and setuptools
-${PIP_COMMAND} install ${PIP_OPTS} ${PIP_INSTALL_OPTIONS} || ${PIP_COMMAND} install ${PIP_OPTS} --isolated ${PIP_INSTALL_OPTIONS}
-
-# Set the constraints now that we know we're using the right version of pip
-PIP_OPTS+=" --constraint global-requirement-pins.txt --constraint ${UPPER_CONSTRAINTS_FILE}"
-
-# Install the required packages for ansible
-$PIP_COMMAND install $PIP_OPTS -r requirements.txt ${ANSIBLE_PACKAGE} || $PIP_COMMAND install --isolated $PIP_OPTS -r requirements.txt ${ANSIBLE_PACKAGE}
+# Install ansible and the other required packages
+${PIP_COMMAND} install ${PIP_OPTS} -r requirements.txt ${ANSIBLE_PACKAGE} \
+  || ${PIP_COMMAND} install --isolated ${PIP_OPTS} -r requirements.txt ${ANSIBLE_PACKAGE}
 
 # Ensure that Ansible binaries run from the venv
 pushd /opt/ansible-runtime/bin

diff --git a/scripts/scripts-library.sh b/scripts/scripts-library.sh
@@ -20,7 +20,6 @@ LINE='----------------------------------------------------------------------'
 MAX_RETRIES=${MAX_RETRIES:-5}
 ANSIBLE_PARAMETERS=${ANSIBLE_PARAMETERS:--e gather_facts=False}
 STARTTIME="${STARTTIME:-$(date +%s)}"
-PIP_INSTALL_OPTIONS=${PIP_INSTALL_OPTIONS:-'pip==9.0.1 setuptools==38.5.0 wheel==0.30.0 '}
 COMMAND_LOGS=${COMMAND_LOGS:-"/openstack/log/ansible_cmd_logs"}
 
 # The default SSHD configuration has MaxSessions = 10. If a deployer changes
@@ -242,41 +241,29 @@ function print_report {
 }
 
 function get_pip {
-
-  # check if pip is already installed
-  if [ "$(which pip)" ]; then
-
-    # make sure that the right pip base packages are installed
-    # If this fails retry with --isolated to bypass the repo server because the repo server will not have
-    # been updated at this point to include any newer pip packages.
-    pip install --upgrade ${PIP_INSTALL_OPTIONS} || pip install --upgrade --isolated ${PIP_INSTALL_OPTIONS}
-
-    # Ensure that our shell knows about the new pip
-    hash -r pip
-
-  # when pip is not installed, install it
+  # The python executable to use when executing get-pip is passed
+  # as a parameter to this function.
+  GETPIP_PYTHON_EXEC_PATH="${1:-$(which python)}"
+
+  # Download the get-pip script using the primary or secondary URL
+  GETPIP_CMD="curl --silent --show-error --retry 5"
+  GETPIP_FILE="/opt/get-pip.py"
+  # If GET_PIP_URL is set, then just use it
+  if [ -n "${GET_PIP_URL:-}" ]; then
+    ${GETPIP_CMD} ${GET_PIP_URL} > ${GETPIP_FILE}
   else
-
-    # Download the get-pip script using the primary or secondary URL
-    GETPIP_CMD="curl --silent --show-error --retry 5"
-    GETPIP_FILE="/opt/get-pip.py"
-    # If GET_PIP_URL is set, then just use it
-    if [ -n "${GET_PIP_URL:-}" ]; then
-      ${GETPIP_CMD} ${GET_PIP_URL} > ${GETPIP_FILE}
-    else
-      # Otherwise, try the two standard URL's
-      ${GETPIP_CMD} https://bootstrap.pypa.io/get-pip.py > ${GETPIP_FILE}\
-        || ${GETPIP_CMD} https://raw.githubusercontent.com/pypa/get-pip/master/get-pip.py > ${GETPIP_FILE}
-    fi
-
-    if head -n 1 /opt/get-pip.py | grep python; then
-      python /opt/get-pip.py ${PIP_INSTALL_OPTIONS}
-      return
-    fi
-
-    echo "A suitable download location for get-pip.py could not be found."
-    exit_fail
+    # Otherwise, try the two standard URL's
+    ${GETPIP_CMD} https://bootstrap.pypa.io/get-pip.py > ${GETPIP_FILE}\
+      || ${GETPIP_CMD} https://raw.githubusercontent.com/pypa/get-pip/master/get-pip.py > ${GETPIP_FILE}
   fi
+
+  ${GETPIP_PYTHON_EXEC_PATH} ${GETPIP_FILE} \
+    pip setuptools wheel \
+    --constraint global-requirement-pins.txt \
+    || ${GETPIP_PYTHON_EXEC_PATH} ${GETPIP_FILE} \
+         pip setuptools wheel \
+         --constraint global-requirement-pins.txt \
+         --isolated
 }
 
 ## Signal traps --------------------------------------------------------------

diff --git a/scripts/sources-branch-updater.sh b/scripts/sources-branch-updater.sh
@@ -254,16 +254,14 @@ done
 
 unset IFS
 
-# Update the PIP_INSTALL_OPTIONS with the current versions of pip, wheel and setuptools
+# Updates global requirement pins for pip, setuptools and wheel
 PIP_CURRENT_OPTIONS=$(./scripts/get-pypi-pkg-version.py -p pip setuptools wheel -l horizontal)
-sed -i.bak "s|^PIP_INSTALL_OPTIONS=.*|PIP_INSTALL_OPTIONS=\$\{PIP_INSTALL_OPTIONS:-'${PIP_CURRENT_OPTIONS}'\}|" scripts/scripts-library.sh
 
 for pin in ${PIP_CURRENT_OPTIONS}; do
   sed -i.bak "s|^$(echo ${pin} | cut -f1 -d=).*|${pin}|" global-requirement-pins.txt
-  sed -i.bak "s|^  - $(echo ${pin} | cut -f1 -d=).*|  - ${pin}|" playbooks/inventory/group_vars/all.yml
 done
 
-echo "Updated pip install options/pins"
+echo "Updated global requirement pins"
 
 # Update the ansible-role-requirements.yml file
 # We don't want to be doing this for the master branch

diff --git a/tests/bootstrap-aio.yml b/tests/bootstrap-aio.yml
@@ -22,6 +22,7 @@
     - role: "pip_install"
     - role: "bootstrap-host"
   vars:
+    ansible_python_interpreter: "/usr/bin/python"
     openstack_confd_entries: "{{ confd_overrides[bootstrap_host_scenario] }}"
     bootstrap_host_scenario: "{{ lookup('env','SCENARIO') | default('aio', true) }}"
     pip_install_upper_constraints_proto: "{{ ansible_python_version | version_compare('2.7.9', '>=') | ternary('https','http') }}"