Skip to content
Permalink
Browse files

Replace ``python-apt`` functionality (#341)

* setup: Add missing ``psutil`` dependency

* Add module with subset of ``apt_pkg`` API

The ``python-apt`` package is a wrapper around the ``apt`` C++
library which is tightly connected to the version of the
distribution it is shipped on.

This in turn makes it incredibly hard to distribute as a wheel for
a charm that supports a large span of distro versions.

We do not want to rely on system installed Python packages but
distribute the direct charm dependencies as part of the charms
wheelhouse.

As the span of distributions we need to support with reactive
charms widens we will run into compability problems with the
current model.

For further reference see discussion in LP: #1824112 and
juju-solutions/layer-basic#135

* Make c-h use provided apt_pkg compability shim

* Detect consumer use of upstream ``apt_pkg`` module

When our consumer use the upstream ``apt_pkg`` module in
conjunction with the apt_cache helper function, they may expect us
to call ``apt_pkg.init()`` for them.

Detect this situation, log a warning and make the call to
``apt_pkg.init()`` to avoid the consumer Python interpreter from
crashing with a segmentation fault.

* Improve string handling in ``upstream_version``

* Update ``apt_cache`` function signature

Make it more clear that any arguments will be ignored.

* Use AttributeDict trick for ``_container`` class

* Accept any kw args to apt_cache for compability
  • Loading branch information...
fnordahl authored and johnsca committed Aug 13, 2019
1 parent b4de1f4 commit d2ea1b8d8c2fb5bc80b8ff2f7f81c82a02bd611b
@@ -39,7 +39,7 @@ userinstall:


.venv:
dpkg-query -W -f='$${status}' gcc python-dev python-virtualenv python-apt 2>/dev/null | grep --invert-match "not-installed" || sudo apt-get install -y python-dev python-virtualenv python-apt
dpkg-query -W -f='$${status}' gcc python-dev python-virtualenv 2>/dev/null | grep --invert-match "not-installed" || sudo apt-get install -y python-dev python-virtualenv
virtualenv .venv --system-site-packages
.venv/bin/pip install -U pip
.venv/bin/pip install -I -r test_requirements.txt
@@ -13,7 +13,6 @@
# limitations under the License.

from __future__ import absolute_import # required for external apt import
from apt import apt_pkg
from six import string_types

from charmhelpers.fetch import (
@@ -26,6 +25,7 @@
WARNING,
)
from charmhelpers.contrib.hardening.audits import BaseAudit
from charmhelpers.fetch import ubuntu_apt_pkg as apt_pkg


class AptConfig(BaseAudit):
@@ -84,7 +84,8 @@
SourceConfigError,
GPGKeyError,
get_upstream_version,
filter_missing_packages
filter_missing_packages,
ubuntu_apt_pkg as apt,
)

from charmhelpers.fetch.snap import (
@@ -443,8 +444,6 @@ def get_os_codename_package(package, fatal=True):
# Second item in list is Version
return line.split()[1]

import apt_pkg as apt

cache = apt_cache()

try:
@@ -658,7 +657,6 @@ def openstack_upgrade_available(package):
a newer version of package.
"""

import apt_pkg as apt
src = config('openstack-origin')
cur_vers = get_os_version_package(package)
if not cur_vers:
@@ -93,7 +93,7 @@ def cmp_pkgrevno(package, revno, pkgcache=None):
the pkgcache argument is None. Be sure to add charmhelpers.fetch if
you call this function, or pass an apt_pkg.Cache() instance.
"""
import apt_pkg
from charmhelpers.fetch import apt_pkg
if not pkgcache:
from charmhelpers.fetch import apt_cache
pkgcache = apt_cache()
@@ -103,6 +103,7 @@ def base_url(self, url):
apt_unhold = fetch.apt_unhold
import_key = fetch.import_key
get_upstream_version = fetch.get_upstream_version
apt_pkg = fetch.ubuntu_apt_pkg
elif __platform__ == "centos":
yum_search = fetch.yum_search

@@ -17,8 +17,9 @@
import platform
import re
import six
import time
import subprocess
import sys
import time

from charmhelpers.core.host import get_distrib_codename

@@ -29,6 +30,7 @@
env_proxy_settings,
)
from charmhelpers.fetch import SourceConfigError, GPGKeyError
from charmhelpers.fetch import ubuntu_apt_pkg

PROPOSED_POCKET = (
"# Proposed\n"
@@ -216,14 +218,28 @@ def filter_missing_packages(packages):
)


def apt_cache(in_memory=True, progress=None):
"""Build and return an apt cache."""
from apt import apt_pkg
apt_pkg.init()
if in_memory:
apt_pkg.config.set("Dir::Cache::pkgcache", "")
apt_pkg.config.set("Dir::Cache::srcpkgcache", "")
return apt_pkg.Cache(progress)
def apt_cache(*_, **__):
"""Shim returning an object simulating the apt_pkg Cache.
:param _: Accept arguments for compability, not used.
:type _: any
:param __: Accept keyword arguments for compability, not used.
:type __: any
:returns:Object used to interrogate the system apt and dpkg databases.
:rtype:ubuntu_apt_pkg.Cache
"""
if 'apt_pkg' in sys.modules:
# NOTE(fnordahl): When our consumer use the upstream ``apt_pkg`` module
# in conjunction with the apt_cache helper function, they may expect us
# to call ``apt_pkg.init()`` for them.
#
# Detect this situation, log a warning and make the call to
# ``apt_pkg.init()`` to avoid the consumer Python interpreter from
# crashing with a segmentation fault.
log('Support for use of upstream ``apt_pkg`` module in conjunction'
'with charm-helpers is deprecated since 2019-06-25', level=WARNING)
sys.modules['apt_pkg'].init()
return ubuntu_apt_pkg.Cache()


def apt_install(packages, options=None, fatal=False):
@@ -723,7 +739,6 @@ def get_upstream_version(package):
@returns None (if not installed) or the upstream version
"""
import apt_pkg
cache = apt_cache()
try:
pkg = cache[package]
@@ -735,4 +750,4 @@ def get_upstream_version(package):
# package is known, but no version is currently installed.
return None

return apt_pkg.upstream_version(pkg.current_ver.ver_str)
return ubuntu_apt_pkg.upstream_version(pkg.current_ver.ver_str)
@@ -0,0 +1,237 @@
# Copyright 2019 Canonical Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Provide a subset of the ``python-apt`` module API.
Data collection is done through subprocess calls to ``apt-cache`` and
``dpkg-query`` commands.
The main purpose for this module is to avoid dependency on the
``python-apt`` python module.
The indicated python module is a wrapper around the ``apt`` C++ library
which is tightly connected to the version of the distribution it was
shipped on. It is not developed in a backward/forward compatible manner.
This in turn makes it incredibly hard to distribute as a wheel for a piece
of python software that supports a span of distro releases [0][1].
Upstream feedback like [2] does not give confidence in this ever changing,
so with this we get rid of the dependency.
0: https://github.com/juju-solutions/layer-basic/pull/135
1: https://bugs.launchpad.net/charm-octavia/+bug/1824112
2: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=845330#10
"""

import locale
import os
import subprocess


class _container(dict):
"""Simple container for attributes."""
__getattr__ = dict.__getitem__
__setattr__ = dict.__setitem__


class Package(_container):
"""Simple container for package attributes."""


class Version(_container):
"""Simple container for version attributes."""


class Cache(object):
"""Simulation of ``apt_pkg`` Cache object."""
def __init__(self, progress=None):
pass

def __getitem__(self, package):
"""Get information about a package from apt and dpkg databases.
:param package: Name of package
:type package: str
:returns: Package object
:rtype: object
:raises: KeyError, subprocess.CalledProcessError
"""
apt_result = self._apt_cache_show([package])[package]
apt_result['name'] = apt_result.pop('package')
pkg = Package(apt_result)
dpkg_result = self._dpkg_list([package]).get(package, {})
current_ver = None
installed_version = dpkg_result.get('version')
if installed_version:
current_ver = Version({'ver_str': installed_version})
pkg.current_ver = current_ver
pkg.architecture = dpkg_result.get('architecture')
return pkg

def _dpkg_list(self, packages):
"""Get data from system dpkg database for package.
:param packages: Packages to get data from
:type packages: List[str]
:returns: Structured data about installed packages, keys like
``dpkg-query --list``
:rtype: dict
:raises: subprocess.CalledProcessError
"""
pkgs = {}
cmd = ['dpkg-query', '--list']
cmd.extend(packages)
if locale.getlocale() == (None, None):
# subprocess calls out to locale.getpreferredencoding(False) to
# determine encoding. Workaround for Trusty where the
# environment appears to not be set up correctly.
locale.setlocale(locale.LC_ALL, 'en_US.UTF-8')
try:
output = subprocess.check_output(cmd,
stderr=subprocess.STDOUT,
universal_newlines=True)
except subprocess.CalledProcessError as cp:
# ``dpkg-query`` may return error and at the same time have
# produced useful output, for example when asked for multiple
# packages where some are not installed
if cp.returncode != 1:
raise
output = cp.output
headings = []
for line in output.splitlines():
if line.startswith('||/'):
headings = line.split()
headings.pop(0)
continue
elif (line.startswith('|') or line.startswith('+') or
line.startswith('dpkg-query:')):
continue
else:
data = line.split(None, 4)
status = data.pop(0)
if status != 'ii':
continue
pkg = {}
pkg.update({k.lower(): v for k, v in zip(headings, data)})
if 'name' in pkg:
pkgs.update({pkg['name']: pkg})
return pkgs

def _apt_cache_show(self, packages):
"""Get data from system apt cache for package.
:param packages: Packages to get data from
:type packages: List[str]
:returns: Structured data about package, keys like
``apt-cache show``
:rtype: dict
:raises: subprocess.CalledProcessError
"""
pkgs = {}
cmd = ['apt-cache', 'show', '--no-all-versions']
cmd.extend(packages)
if locale.getlocale() == (None, None):
# subprocess calls out to locale.getpreferredencoding(False) to
# determine encoding. Workaround for Trusty where the
# environment appears to not be set up correctly.
locale.setlocale(locale.LC_ALL, 'en_US.UTF-8')
try:
output = subprocess.check_output(cmd,
stderr=subprocess.STDOUT,
universal_newlines=True)
previous = None
pkg = {}
for line in output.splitlines():
if not line:
if 'package' in pkg:
pkgs.update({pkg['package']: pkg})
pkg = {}
continue
if line.startswith(' '):
if previous and previous in pkg:
pkg[previous] += os.linesep + line.lstrip()
continue
if ':' in line:
kv = line.split(':', 1)
key = kv[0].lower()
if key == 'n':
continue
previous = key
pkg.update({key: kv[1].lstrip()})
except subprocess.CalledProcessError as cp:
# ``apt-cache`` returns 100 if none of the packages asked for
# exist in the apt cache.
if cp.returncode != 100:
raise
return pkgs


def init():
"""Compability shim that does nothing."""
pass


def upstream_version(version):
"""Extracts upstream version from a version string.
Upstream reference: https://salsa.debian.org/apt-team/apt/blob/master/
apt-pkg/deb/debversion.cc#L259
:param version: Version string
:type version: str
:returns: Upstream version
:rtype: str
"""
if version:
version = version.split(':')[-1]
version = version.split('-')[0]
return version


def version_compare(a, b):
"""Compare the given versions.
Call out to ``dpkg`` to make sure the code doing the comparison is
compatible with what the ``apt`` library would do. Mimic the return
values.
Upstream reference:
https://apt-team.pages.debian.net/python-apt/library/apt_pkg.html
?highlight=version_compare#apt_pkg.version_compare
:param a: version string
:type a: str
:param b: version string
:type b: str
:returns: >0 if ``a`` is greater than ``b``, 0 if a equals b,
<0 if ``a`` is smaller than ``b``
:rtype: int
:raises: subprocess.CalledProcessError, RuntimeError
"""
for op in ('gt', 1), ('eq', 0), ('lt', -1):
try:
subprocess.check_call(['dpkg', '--compare-versions',
a, op[0], b],
stderr=subprocess.STDOUT,
universal_newlines=True)
return op[1]
except subprocess.CalledProcessError as cp:
if cp.returncode == 1:
continue
raise
else:
raise RuntimeError('Unable to compare "{}" and "{}", according to '
'our logic they are neither greater, equal nor '
'less than each other.')
@@ -35,6 +35,7 @@
'Tempita',
'Jinja2',
'six',
'psutil',
],
'packages': find_packages(exclude=('tests', 'tests.*', 'tools', 'tools.*')),
'scripts': [
@@ -1,7 +1,6 @@
# Test-only dependencies are unpinned.
#
git+https://git.launchpad.net/ubuntu/+source/python-distutils-extra
git+https://git.launchpad.net/python-apt@1.1.y-xenial
pip
six
coverage>=3.6

0 comments on commit d2ea1b8

Please sign in to comment.
You can’t perform that action at this time.