From 77fbc32ddf843d2301037fdba4e9639b55c276ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erik=20L=C3=B6nroth?= Date: Thu, 21 Nov 2019 12:11:56 +0100 Subject: [PATCH 1/5] Fixed the support for centos and rhel --- lib/charms/layer/basic.py | 124 ++++++++++++++++++++++++++++++++------ 1 file changed, 106 insertions(+), 18 deletions(-) diff --git a/lib/charms/layer/basic.py b/lib/charms/layer/basic.py index 3931959..8017f27 100644 --- a/lib/charms/layer/basic.py +++ b/lib/charms/layer/basic.py @@ -1,5 +1,6 @@ import os import sys +import re import shutil from glob import glob from subprocess import check_call, check_output, CalledProcessError @@ -8,15 +9,49 @@ from charms import layer from charms.layer.execd import execd_preinstall +def get_series(): + """ + Return series for a few known OS:es. + Tested as of 2019 november: + * centos6, centos7, rhel6. + * bionic + """ + series = "" + + # Looking for content in /etc/os-release + # works for ubuntu + some centos + if os.path.isfile('/etc/os-release'): + d = {} + with open('/etc/os-release', 'r') as rel: + for l in rel: + if not re.match(r'^\s*$', l): + k, v = l.split('=') + d[k.strip()] = v.strip().replace('"', '') + series = "{ID}{VERSION_ID}".format(**d) + + # Looking for content in /etc/redhat-release + # works for redhat enterprise systems + elif os.path.isfile('/etc/redhat-release'): + with open('/etc/redhat-release', 'r') as redhatlsb: + # CentOS Linux release 7.7.1908 (Core) + l = redhatlsb.readline() + release = int(l.split("release")[1].split()[0][0]) + series = "centos" + str(release) + + # Looking for content in /etc/lsb-release + # works for ubuntu + elif os.path.isfile('/etc/lsb-release'): + d = {} + with open('/etc/lsb-release', 'r') as lsb: + for l in lsb: + k, v = l.split('=') + d[k.strip()] = v.strip() + series = d['DISTRIB_CODENAME'] -def lsb_release(): - """Return /etc/lsb-release in a dict""" - d = {} - with open('/etc/lsb-release', 'r') as lsb: - for l in lsb: - k, v = l.split('=') - d[k.strip()] = v.strip() - return d + # This is what happens if we cant figure out the OS. + else: + series = "unknown" + return series def bootstrap_charm_deps(): @@ -30,6 +65,30 @@ def bootstrap_charm_deps(): # unless the operator has created and populated $JUJU_CHARM_DIR/exec.d. execd_preinstall() # ensure that $JUJU_CHARM_DIR/bin is on the path, for helper scripts + + series = get_series() + + # OMG?! is build-essentials needed? + ubuntu_packages = ['python3-pip', + 'python3-setuptools', + 'python3-yaml', + 'python3-dev', + 'python3-wheel', + 'build-essential'] + + # I'm not going to "yum group info "Development Tools" + # omitting above madness + centos_packages = ['python3-pip', + 'python3-setuptools', + 'python3-devel', + 'python3-wheel' ] + + packages_needed = [] + if 'centos' in series: + packages_needed = centos_packages + else: + packages_needed = ubuntu_packages + charm_dir = os.environ['JUJU_CHARM_DIR'] os.environ['PATH'] += ':%s' % os.path.join(charm_dir, 'bin') venv = os.path.abspath('../.venv') @@ -71,24 +130,25 @@ def bootstrap_charm_deps(): "allow_hosts = ''\n", "find_links = file://{}/wheelhouse/\n".format(charm_dir), ]) - apt_install([ - 'python3-pip', - 'python3-setuptools', - 'python3-yaml', - 'python3-dev', - 'python3-wheel', - 'build-essential', - ]) + if 'centos' in series: + yum_install(packages_needed) + else: + apt_install(packages_needed) from charms.layer import options cfg = options.get('basic') # include packages defined in layer.yaml - apt_install(cfg.get('packages', [])) + if 'centos' in series: + yum_install(cfg.get('packages', [])) + else: + apt_install(cfg.get('packages', [])) # if we're using a venv, set it up if cfg.get('use_venv'): if not os.path.exists(venv): - series = lsb_release()['DISTRIB_CODENAME'] + series = get_series() if series in ('precise', 'trusty'): apt_install(['python-virtualenv']) + elif 'centos' in series: + yum_install(['python-virtualenv']) else: apt_install(['virtualenv']) cmd = ['virtualenv', '-ppython3', '--never-download', venv] @@ -117,6 +177,12 @@ def bootstrap_charm_deps(): '-f', 'wheelhouse'] + glob('wheelhouse/*')) # re-enable installation from pypi os.remove('/root/.pydistutils.cfg') + + # install pyyaml for centos7 altough this can be made through cloud-init + # Info : https://discourse.jujucharms.com/t/charms-for-centos-lets-begin/2339/8?u=erik-lonroth + if 'centos' in series: + check_call([pip, 'install', '-U', 'pyyaml']) + # install python packages from layer options if cfg.get('python_packages'): check_call([pip, 'install', '-U'] + cfg.get('python_packages')) @@ -250,6 +316,28 @@ def apt_install(packages): else: break +def yum_install(packages): + """ Installs packages with yum. + This function largely mimics the apt_install function for consistency. + """ + if packages: + env = os.environ.copy() + cmd = ['yum','-y','install'] + for attempt in range(3): + try: + check_call(cmd + packages, env=env) + except CalledProcessError: + if attempt == 2: + raise + try: + check_call(['yum', 'update']) + except CalledProcessError: + pass + sleep(5) + else: + break + else: + pass def init_config_states(): import yaml From 6216f7e7f5621007fdbdb236e5882c821a19510f Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Fri, 31 Jan 2020 13:51:46 +0000 Subject: [PATCH 2/5] Pin Jinja2/PyYAML wheelhouse.txt and add eoan test Pin Jinja2/PyYAML in wheelhouse.txt for Python 3.4 The last versions of Jinja2 and PyYAML to support Python 3.4 are 2.10.1 and 5.2, respectively. In order to continue supporting Ubuntu Trusty, which defaults to Python 3.4, the wheelhouse.txt is updated to ensure newer versions aren't used. The disco test is also dropped and replaced with eoan. Fixes #152 --- tests/bundles/minimal.yaml | 4 ++-- tests/charm-minimal/metadata.yaml | 2 +- wheelhouse.txt | 3 +++ 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/bundles/minimal.yaml b/tests/bundles/minimal.yaml index 1d716a9..fdbc109 100644 --- a/tests/bundles/minimal.yaml +++ b/tests/bundles/minimal.yaml @@ -11,7 +11,7 @@ applications: series: bionic charm: /tmp/charm-builds/minimal num_units: 1 - minimal-disco: - series: disco + minimal-eoan: + series: eoan charm: /tmp/charm-builds/minimal num_units: 1 diff --git a/tests/charm-minimal/metadata.yaml b/tests/charm-minimal/metadata.yaml index 1728759..b8b6974 100644 --- a/tests/charm-minimal/metadata.yaml +++ b/tests/charm-minimal/metadata.yaml @@ -8,4 +8,4 @@ series: - trusty - xenial - bionic - - disco + - eoan diff --git a/wheelhouse.txt b/wheelhouse.txt index 664a385..3da4cab 100644 --- a/wheelhouse.txt +++ b/wheelhouse.txt @@ -2,6 +2,9 @@ # even with installing setuptools before upgrading pip ends up with pip seeing # the older setuptools at the system level if include_system_packages is true pip>=18.1,<19.0 +# pin Jinja2 and PyYAML to the last versions supporting python 3.4 for trusty +Jinja2<=2.10.1 +PyYAML<=5.2 setuptools<42 setuptools-scm<=1.17.0 charmhelpers>=0.4.0,<1.0.0 From b6d9996599a8dee1311243d34278c17171b6338a Mon Sep 17 00:00:00 2001 From: Cory Johns Date: Wed, 19 Feb 2020 13:22:54 -0500 Subject: [PATCH 3/5] Fix lint errors --- lib/charms/layer/basic.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/lib/charms/layer/basic.py b/lib/charms/layer/basic.py index 8017f27..4e8a760 100644 --- a/lib/charms/layer/basic.py +++ b/lib/charms/layer/basic.py @@ -9,6 +9,7 @@ from charms import layer from charms.layer.execd import execd_preinstall + def get_series(): """ Return series for a few known OS:es. @@ -34,8 +35,8 @@ def get_series(): elif os.path.isfile('/etc/redhat-release'): with open('/etc/redhat-release', 'r') as redhatlsb: # CentOS Linux release 7.7.1908 (Core) - l = redhatlsb.readline() - release = int(l.split("release")[1].split()[0][0]) + line = redhatlsb.readline() + release = int(line.split("release")[1].split()[0][0]) series = "centos" + str(release) # Looking for content in /etc/lsb-release @@ -81,7 +82,7 @@ def bootstrap_charm_deps(): centos_packages = ['python3-pip', 'python3-setuptools', 'python3-devel', - 'python3-wheel' ] + 'python3-wheel'] packages_needed = [] if 'centos' in series: @@ -178,8 +179,9 @@ def bootstrap_charm_deps(): # re-enable installation from pypi os.remove('/root/.pydistutils.cfg') - # install pyyaml for centos7 altough this can be made through cloud-init - # Info : https://discourse.jujucharms.com/t/charms-for-centos-lets-begin/2339/8?u=erik-lonroth + # install pyyaml for centos7, since, unlike the ubuntu image, the + # default image for centos doesn't include pyyaml; see the discussion: + # https://discourse.jujucharms.com/t/charms-for-centos-lets-begin if 'centos' in series: check_call([pip, 'install', '-U', 'pyyaml']) @@ -316,13 +318,14 @@ def apt_install(packages): else: break + def yum_install(packages): """ Installs packages with yum. This function largely mimics the apt_install function for consistency. """ if packages: env = os.environ.copy() - cmd = ['yum','-y','install'] + cmd = ['yum', '-y', 'install'] for attempt in range(3): try: check_call(cmd + packages, env=env) @@ -339,6 +342,7 @@ def yum_install(packages): else: pass + def init_config_states(): import yaml from charmhelpers.core import hookenv From 43dee0250e062e3407964f78271c93e3f1045f7f Mon Sep 17 00:00:00 2001 From: Cory Johns Date: Wed, 19 Feb 2020 13:57:54 -0500 Subject: [PATCH 4/5] Fix precise / trusty detection This PR included a change which preferred `/etc/os-release` over `/etc/lsb-release` which changed the format of the series which was returned. --- lib/charms/layer/basic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/charms/layer/basic.py b/lib/charms/layer/basic.py index 4e8a760..a2518e1 100644 --- a/lib/charms/layer/basic.py +++ b/lib/charms/layer/basic.py @@ -146,7 +146,7 @@ def bootstrap_charm_deps(): if cfg.get('use_venv'): if not os.path.exists(venv): series = get_series() - if series in ('precise', 'trusty'): + if series in ('ubuntu12.04', 'precise', 'ubuntu14.04', 'trusty'): apt_install(['python-virtualenv']) elif 'centos' in series: yum_install(['python-virtualenv']) From f06bbc4d4fb45e5a20774ad841854eaac9cb7650 Mon Sep 17 00:00:00 2001 From: Cory Johns Date: Wed, 19 Feb 2020 14:09:11 -0500 Subject: [PATCH 5/5] Fix line-too-long lint error. --- lib/charms/layer/basic.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/charms/layer/basic.py b/lib/charms/layer/basic.py index a2518e1..bc18121 100644 --- a/lib/charms/layer/basic.py +++ b/lib/charms/layer/basic.py @@ -146,7 +146,8 @@ def bootstrap_charm_deps(): if cfg.get('use_venv'): if not os.path.exists(venv): series = get_series() - if series in ('ubuntu12.04', 'precise', 'ubuntu14.04', 'trusty'): + if series in ('ubuntu12.04', 'precise', + 'ubuntu14.04', 'trusty'): apt_install(['python-virtualenv']) elif 'centos' in series: yum_install(['python-virtualenv'])