Merge pull request canonical#160 from juju-solutions/bug/159/focal-pip

Fix broken pip due to downgrade on Focal+

.travis.yml

@@ -33,6 +33,7 @@ script:
       sudo sh -c 'echo PATH=/snap/bin:$PATH >> /etc/environment';
       sudo lxd waitready;
       sudo lxd init --auto;
+      cat .travis/profile-update.yaml | sudo lxc profile edit default;
       sudo usermod -a -G lxd travis;
       sudo su - travis -c 'juju bootstrap --no-gui localhost';
     fi
@@ -43,5 +44,5 @@ script:
     fi
 after_failure:
   - if [ $ENV = 'func' ]; then
-      sudo su travis -c 'for application in trusty xenial bionic cosmic disco; do juju ssh -m $(juju models --format yaml|grep "^- name:.*zaza"|cut -f2 -d/) minimal-${application}/0 "sudo cat /var/log/juju/unit*.log";done';
+      sudo su travis -c 'for application in trusty xenial bionic eoan focal; do juju ssh -m $(juju models --format yaml|grep "^- name:.*zaza"|cut -f2 -d/) minimal-${application}/0 "sudo cat /var/log/juju/unit*.log";done';
     fi

.travis/profile-update.yaml

@@ -0,0 +1,12 @@
+config: {}
+description: Default LXD profile - updated
+devices:
+  eth0:
+    name: eth0
+    parent: lxdbr0
+    nictype: bridged
+    type: nic
+  root:
+    path: /
+    pool: default
+    type: disk

lib/charms/layer/basic.py

@@ -2,6 +2,7 @@
 import sys
 import re
 import shutil
+from distutils.version import LooseVersion
 from glob import glob
 from subprocess import check_call, check_output, CalledProcessError
 from time import sleep
@@ -165,18 +166,20 @@ def bootstrap_charm_deps():
             # from changing it
             if os.path.exists('/usr/bin/pip'):
                 shutil.copy2('/usr/bin/pip', '/usr/bin/pip.save')
-        # need newer pip, to fix spurious Double Requirement error:
-        # https://github.com/pypa/pip/issues/56
-        check_call([pip, 'install', '-U', '--no-index', '-f', 'wheelhouse',
-                    'pip'])
-        # per https://github.com/juju-solutions/layer-basic/issues/110
-        # this replaces the setuptools that was copied over from the system on
-        # venv create with latest setuptools and adds setuptools_scm
-        check_call([pip, 'install', '-U', '--no-index', '-f', 'wheelhouse',
-                    'setuptools', 'setuptools-scm'])
-        # install the rest of the wheelhouse deps
+        pre_install_pkgs = ['pip', 'setuptools', 'setuptools-scm']
+        # we bundle these packages to work around bugs in older versions (such
+        # as https://github.com/pypa/pip/issues/56), but if the system already
+        # provided a newer version, downgrading it can cause other problems
+        _update_if_newer(pip, pre_install_pkgs)
+        # install the rest of the wheelhouse deps (extract the pkg names into
+        # a set so that we can ignore the pre-install packages and let pip
+        # choose the best version in case there are multiple from layer
+        # conflicts)
+        pkgs = {os.path.basename(wheel).split('-')[0].replace('_', '-')
+                for wheel in glob('wheelhouse/*')}
+        pkgs -= set(pre_install_pkgs)
         check_call([pip, 'install', '-U', '--ignore-installed', '--no-index',
-                   '-f', 'wheelhouse'] + glob('wheelhouse/*'))
+                   '-f', 'wheelhouse'] + list(pkgs))
         # re-enable installation from pypi
         os.remove('/root/.pydistutils.cfg')
 
@@ -220,6 +223,33 @@ def bootstrap_charm_deps():
         reload_interpreter(vpy if cfg.get('use_venv') else sys.argv[0])
 
 
+def _load_installed_versions(pip):
+    pip_freeze = check_output([pip, 'freeze']).decode('utf8')
+    versions = {}
+    for pkg_ver in pip_freeze.splitlines():
+        pkg, ver = pkg_ver.split('==')
+        versions[pkg] = LooseVersion(ver)
+    return versions
+
+
+def _load_wheelhouse_versions():
+    versions = {}
+    for wheel in glob('wheelhouse/*'):
+        pkg, ver = os.path.basename(wheel).split('-')
+        # nb: LooseVersion ignores the file extension
+        versions[pkg.replace('_', '-')] = LooseVersion(ver)
+    return versions
+
+
+def _update_if_newer(pip, pkgs):
+    installed = _load_installed_versions(pip)
+    wheelhouse = _load_wheelhouse_versions()
+    for pkg in pkgs:
+        if pkg not in installed or wheelhouse[pkg] > installed[pkg]:
+            check_call([pip, 'install', '-U', '--no-index', '-f', 'wheelhouse',
+                        pkg])
+
+
 def install_or_update_charm_env():
     # On Trusty python3-pkg-resources is not installed
     try:

tests/bundles/minimal.yaml

@@ -15,3 +15,7 @@ applications:
     series: eoan
     charm: /tmp/charm-builds/minimal
     num_units: 1
+  minimal-focal:
+    series: focal
+    charm: /tmp/charm-builds/minimal
+    num_units: 1

tests/charm-minimal/metadata.yaml

@@ -9,3 +9,4 @@ series:
   - xenial
   - bionic
   - eoan
+  - focal

tests/tests.yaml

@@ -3,3 +3,6 @@ gate_bundles:
 - minimal
 tests:
 - zaza.charm_tests.noop.tests.NoopTest
+test_options:
+  force_deploy:
+    - minimal