From 4761d6c966966441b8eb388af152b25ddcde6636 Mon Sep 17 00:00:00 2001 From: Cory Johns Date: Fri, 3 Apr 2020 12:34:12 -0400 Subject: [PATCH] Merge pull request #160 from juju-solutions/bug/159/focal-pip Fix broken pip due to downgrade on Focal+ --- .travis.yml | 3 +- .travis/profile-update.yaml | 12 +++++++ lib/charms/layer/basic.py | 52 ++++++++++++++++++++++++------- tests/bundles/minimal.yaml | 4 +++ tests/charm-minimal/metadata.yaml | 1 + tests/tests.yaml | 3 ++ 6 files changed, 63 insertions(+), 12 deletions(-) create mode 100644 .travis/profile-update.yaml diff --git a/.travis.yml b/.travis.yml index c04e5d8..5244b09 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,6 +33,7 @@ script: sudo sh -c 'echo PATH=/snap/bin:$PATH >> /etc/environment'; sudo lxd waitready; sudo lxd init --auto; + cat .travis/profile-update.yaml | sudo lxc profile edit default; sudo usermod -a -G lxd travis; sudo su - travis -c 'juju bootstrap --no-gui localhost'; fi @@ -43,5 +44,5 @@ script: fi after_failure: - if [ $ENV = 'func' ]; then - sudo su travis -c 'for application in trusty xenial bionic cosmic disco; do juju ssh -m $(juju models --format yaml|grep "^- name:.*zaza"|cut -f2 -d/) minimal-${application}/0 "sudo cat /var/log/juju/unit*.log";done'; + sudo su travis -c 'for application in trusty xenial bionic eoan focal; do juju ssh -m $(juju models --format yaml|grep "^- name:.*zaza"|cut -f2 -d/) minimal-${application}/0 "sudo cat /var/log/juju/unit*.log";done'; fi diff --git a/.travis/profile-update.yaml b/.travis/profile-update.yaml new file mode 100644 index 0000000..57f96eb --- /dev/null +++ b/.travis/profile-update.yaml @@ -0,0 +1,12 @@ +config: {} +description: Default LXD profile - updated +devices: + eth0: + name: eth0 + parent: lxdbr0 + nictype: bridged + type: nic + root: + path: / + pool: default + type: disk diff --git a/lib/charms/layer/basic.py b/lib/charms/layer/basic.py index bc18121..d3b06ac 100644 --- a/lib/charms/layer/basic.py +++ b/lib/charms/layer/basic.py @@ -2,6 +2,7 @@ import sys import re import shutil +from distutils.version import LooseVersion from glob import glob from subprocess import check_call, check_output, CalledProcessError from time import sleep @@ -165,18 +166,20 @@ def bootstrap_charm_deps(): # from changing it if os.path.exists('/usr/bin/pip'): shutil.copy2('/usr/bin/pip', '/usr/bin/pip.save') - # need newer pip, to fix spurious Double Requirement error: - # https://github.com/pypa/pip/issues/56 - check_call([pip, 'install', '-U', '--no-index', '-f', 'wheelhouse', - 'pip']) - # per https://github.com/juju-solutions/layer-basic/issues/110 - # this replaces the setuptools that was copied over from the system on - # venv create with latest setuptools and adds setuptools_scm - check_call([pip, 'install', '-U', '--no-index', '-f', 'wheelhouse', - 'setuptools', 'setuptools-scm']) - # install the rest of the wheelhouse deps + pre_install_pkgs = ['pip', 'setuptools', 'setuptools-scm'] + # we bundle these packages to work around bugs in older versions (such + # as https://github.com/pypa/pip/issues/56), but if the system already + # provided a newer version, downgrading it can cause other problems + _update_if_newer(pip, pre_install_pkgs) + # install the rest of the wheelhouse deps (extract the pkg names into + # a set so that we can ignore the pre-install packages and let pip + # choose the best version in case there are multiple from layer + # conflicts) + pkgs = {os.path.basename(wheel).split('-')[0].replace('_', '-') + for wheel in glob('wheelhouse/*')} + pkgs -= set(pre_install_pkgs) check_call([pip, 'install', '-U', '--ignore-installed', '--no-index', - '-f', 'wheelhouse'] + glob('wheelhouse/*')) + '-f', 'wheelhouse'] + list(pkgs)) # re-enable installation from pypi os.remove('/root/.pydistutils.cfg') @@ -220,6 +223,33 @@ def bootstrap_charm_deps(): reload_interpreter(vpy if cfg.get('use_venv') else sys.argv[0]) +def _load_installed_versions(pip): + pip_freeze = check_output([pip, 'freeze']).decode('utf8') + versions = {} + for pkg_ver in pip_freeze.splitlines(): + pkg, ver = pkg_ver.split('==') + versions[pkg] = LooseVersion(ver) + return versions + + +def _load_wheelhouse_versions(): + versions = {} + for wheel in glob('wheelhouse/*'): + pkg, ver = os.path.basename(wheel).split('-') + # nb: LooseVersion ignores the file extension + versions[pkg.replace('_', '-')] = LooseVersion(ver) + return versions + + +def _update_if_newer(pip, pkgs): + installed = _load_installed_versions(pip) + wheelhouse = _load_wheelhouse_versions() + for pkg in pkgs: + if pkg not in installed or wheelhouse[pkg] > installed[pkg]: + check_call([pip, 'install', '-U', '--no-index', '-f', 'wheelhouse', + pkg]) + + def install_or_update_charm_env(): # On Trusty python3-pkg-resources is not installed try: diff --git a/tests/bundles/minimal.yaml b/tests/bundles/minimal.yaml index fdbc109..5666bbf 100644 --- a/tests/bundles/minimal.yaml +++ b/tests/bundles/minimal.yaml @@ -15,3 +15,7 @@ applications: series: eoan charm: /tmp/charm-builds/minimal num_units: 1 + minimal-focal: + series: focal + charm: /tmp/charm-builds/minimal + num_units: 1 diff --git a/tests/charm-minimal/metadata.yaml b/tests/charm-minimal/metadata.yaml index b8b6974..6dbe2f4 100644 --- a/tests/charm-minimal/metadata.yaml +++ b/tests/charm-minimal/metadata.yaml @@ -9,3 +9,4 @@ series: - xenial - bionic - eoan + - focal diff --git a/tests/tests.yaml b/tests/tests.yaml index 5f555e0..f41d751 100644 --- a/tests/tests.yaml +++ b/tests/tests.yaml @@ -3,3 +3,6 @@ gate_bundles: - minimal tests: - zaza.charm_tests.noop.tests.NoopTest +test_options: + force_deploy: + - minimal