Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Replace ``python-apt`` functionality #341

Merged
merged 8 commits into from Aug 13, 2019
@@ -39,7 +39,7 @@ userinstall:


.venv:
dpkg-query -W -f='$${status}' gcc python-dev python-virtualenv python-apt 2>/dev/null | grep --invert-match "not-installed" || sudo apt-get install -y python-dev python-virtualenv python-apt
dpkg-query -W -f='$${status}' gcc python-dev python-virtualenv 2>/dev/null | grep --invert-match "not-installed" || sudo apt-get install -y python-dev python-virtualenv
virtualenv .venv --system-site-packages
.venv/bin/pip install -U pip
.venv/bin/pip install -I -r test_requirements.txt
@@ -13,7 +13,6 @@
# limitations under the License.

from __future__ import absolute_import # required for external apt import
from apt import apt_pkg
from six import string_types

from charmhelpers.fetch import (
@@ -26,6 +25,7 @@
WARNING,
)
from charmhelpers.contrib.hardening.audits import BaseAudit
from charmhelpers.fetch import ubuntu_apt_pkg as apt_pkg


class AptConfig(BaseAudit):
@@ -84,7 +84,8 @@
SourceConfigError,
GPGKeyError,
get_upstream_version,
filter_missing_packages
filter_missing_packages,
ubuntu_apt_pkg as apt,
)

from charmhelpers.fetch.snap import (
@@ -443,8 +444,6 @@ def get_os_codename_package(package, fatal=True):
# Second item in list is Version
return line.split()[1]

import apt_pkg as apt

cache = apt_cache()

try:
@@ -658,7 +657,6 @@ def openstack_upgrade_available(package):
a newer version of package.
"""

import apt_pkg as apt
src = config('openstack-origin')
cur_vers = get_os_version_package(package)
if not cur_vers:
@@ -93,7 +93,7 @@ def cmp_pkgrevno(package, revno, pkgcache=None):
the pkgcache argument is None. Be sure to add charmhelpers.fetch if
you call this function, or pass an apt_pkg.Cache() instance.
"""
import apt_pkg
from charmhelpers.fetch import apt_pkg
if not pkgcache:
from charmhelpers.fetch import apt_cache
pkgcache = apt_cache()
@@ -103,6 +103,7 @@ def base_url(self, url):
apt_unhold = fetch.apt_unhold
import_key = fetch.import_key
get_upstream_version = fetch.get_upstream_version
apt_pkg = fetch.ubuntu_apt_pkg
elif __platform__ == "centos":
yum_search = fetch.yum_search

@@ -17,8 +17,9 @@
import platform
import re
import six
import time
import subprocess
import sys
import time

from charmhelpers.core.host import get_distrib_codename

@@ -29,6 +30,7 @@
env_proxy_settings,
)
from charmhelpers.fetch import SourceConfigError, GPGKeyError
from charmhelpers.fetch import ubuntu_apt_pkg

PROPOSED_POCKET = (
"# Proposed\n"
@@ -217,13 +219,25 @@ def filter_missing_packages(packages):


def apt_cache(in_memory=True, progress=None):
"""Build and return an apt cache."""
from apt import apt_pkg
apt_pkg.init()
if in_memory:
apt_pkg.config.set("Dir::Cache::pkgcache", "")
apt_pkg.config.set("Dir::Cache::srcpkgcache", "")
return apt_pkg.Cache(progress)
"""Shim returning an object simulating the apt_pkg Cache.
:param in_memory: Kept for compability reasons, has no effect.
This conversation was marked as resolved by fnordahl

This comment has been minimized.

Copy link
@ajkavanagh

ajkavanagh Aug 13, 2019

Contributor

You could just use *args in the function signature to show that there might be args, but that you are ignoring them. (or even *_ which would make it even more obvious)?

This comment has been minimized.

Copy link
@fnordahl

fnordahl Aug 13, 2019

Author Member

👍

:param progress: Kept for compability reasons, has no effect.
:returns:Object used to interrogate the system apt and dpkg databases.
:rtype:ubuntu_apt_pkg.Cache
"""
if 'apt_pkg' in sys.modules:
# NOTE(fnordahl): When our consumer use the upstream ``apt_pkg`` module
# in conjunction with the apt_cache helper function, they may expect us
# to call ``apt_pkg.init()`` for them.
#
# Detect this situation, log a warning and make the call to
# ``apt_pkg.init()`` to avoid the consumer Python interpreter from
# crashing with a segmentation fault.
log('Support for use of upstream ``apt_pkg`` module in conjunction'
'with charm-helpers is deprecated since 2019-06-25', level=WARNING)
sys.modules['apt_pkg'].init()
return ubuntu_apt_pkg.Cache()


def apt_install(packages, options=None, fatal=False):
@@ -723,7 +737,6 @@ def get_upstream_version(package):
@returns None (if not installed) or the upstream version
"""
import apt_pkg
cache = apt_cache()
try:
pkg = cache[package]
@@ -735,4 +748,4 @@ def get_upstream_version(package):
# package is known, but no version is currently installed.
return None

return apt_pkg.upstream_version(pkg.current_ver.ver_str)
return ubuntu_apt_pkg.upstream_version(pkg.current_ver.ver_str)
@@ -0,0 +1,245 @@
# Copyright 2019 Canonical Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Provide a subset of the ``python-apt`` module API.
Data collection is done through subprocess calls to ``apt-cache`` and
``dpkg-query`` commands.
The main purpose for this module is to avoid dependency on the
``python-apt`` python module.
The indicated python module is a wrapper around the ``apt`` C++ library
which is tightly connected to the version of the distribution it was
shipped on. It is not developed in a backward/forward compatible manner.
This in turn makes it incredibly hard to distribute as a wheel for a piece
of python software that supports a span of distro releases [0][1].
Upstream feedback like [2] does not give confidence in this ever changing,
so with this we get rid of the dependency.
0: https://github.com/juju-solutions/layer-basic/pull/135
1: https://bugs.launchpad.net/charm-octavia/+bug/1824112
2: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=845330#10
"""

import locale
import os
import subprocess


class _container(object):
"""Simple container for attributes."""
def __init__(self, attr_map):
"""Initialize package attribute container.
:param attr_map: Dictionary key value pairs to transform into
attributes with a value on instance of the class.
:type attr_map: Dict[str, str]
"""
for k, v in attr_map.items():
setattr(self, k, v)
This conversation was marked as resolved by fnordahl

This comment has been minimized.

Copy link
@ajkavanagh

ajkavanagh Aug 13, 2019

Contributor

This is a cute way of almost getting the same thing:

class AttributeDict(dict): 
    __getattr__ = dict.__getitem__
    __setattr__ = dict.__setitem__

i.e. make x.attr work like x['attr']. However, you may be aiming to achieve another objective.

This comment has been minimized.

Copy link
@fnordahl

fnordahl Aug 13, 2019

Author Member

That's very neat, thank you for the tip!



class Package(_container):
"""Simple container for package attributes."""


class Version(_container):
"""Simple container for version attributes."""


class Cache(object):
"""Simulation of ``apt_pkg`` Cache object."""
def __init__(self, progress=None):
pass

def __getitem__(self, package):
"""Get information about a package from apt and dpkg databases.
:param package: Name of package
:type package: str
:returns: Package object
:rtype: object
:raises: KeyError, subprocess.CalledProcessError
"""
apt_result = self._apt_cache_show([package])[package]
apt_result['name'] = apt_result.pop('package')
pkg = Package(apt_result)
dpkg_result = self._dpkg_list([package]).get(package, {})
current_ver = None
installed_version = dpkg_result.get('version')
if installed_version:
current_ver = Version({'ver_str': installed_version})
pkg.current_ver = current_ver
pkg.architecture = dpkg_result.get('architecture')
return pkg

def _dpkg_list(self, packages):
"""Get data from system dpkg database for package.
:param packages: Packages to get data from
:type packages: List[str]
:returns: Structured data about installed packages, keys like
``dpkg-query --list``
:rtype: dict
:raises: subprocess.CalledProcessError
"""
pkgs = {}
cmd = ['dpkg-query', '--list']
cmd.extend(packages)
if locale.getlocale() == (None, None):
# subprocess calls out to locale.getpreferredencoding(False) to
# determine encoding. Workaround for Trusty where the
# environment appears to not be set up correctly.
locale.setlocale(locale.LC_ALL, 'en_US.UTF-8')
try:
output = subprocess.check_output(cmd,
stderr=subprocess.STDOUT,
universal_newlines=True)
except subprocess.CalledProcessError as cp:
# ``dpkg-query`` may return error and at the same time have
# produced useful output, for example when asked for multiple
# packages where some are not installed
if cp.returncode != 1:
raise
output = cp.output
headings = []
for line in output.splitlines():
if line.startswith('||/'):
headings = line.split()
headings.pop(0)
continue
elif (line.startswith('|') or line.startswith('+') or
line.startswith('dpkg-query:')):
continue
else:
data = line.split(None, 4)
status = data.pop(0)
if status != 'ii':
continue
pkg = {}
pkg.update({k.lower(): v for k, v in zip(headings, data)})
if 'name' in pkg:
pkgs.update({pkg['name']: pkg})
return pkgs

def _apt_cache_show(self, packages):
"""Get data from system apt cache for package.
:param packages: Packages to get data from
:type packages: List[str]
:returns: Structured data about package, keys like
``apt-cache show``
:rtype: dict
:raises: subprocess.CalledProcessError
"""
pkgs = {}
cmd = ['apt-cache', 'show', '--no-all-versions']
cmd.extend(packages)
if locale.getlocale() == (None, None):
# subprocess calls out to locale.getpreferredencoding(False) to
# determine encoding. Workaround for Trusty where the
# environment appears to not be set up correctly.
locale.setlocale(locale.LC_ALL, 'en_US.UTF-8')
try:
output = subprocess.check_output(cmd,
stderr=subprocess.STDOUT,
universal_newlines=True)
previous = None
pkg = {}
for line in output.splitlines():
if not line:
if 'package' in pkg:
pkgs.update({pkg['package']: pkg})
pkg = {}
continue
if line.startswith(' '):
if previous and previous in pkg:
pkg[previous] += os.linesep + line.lstrip()
continue
if ':' in line:
kv = line.split(':', 1)
key = kv[0].lower()
if key == 'n':
continue
previous = key
pkg.update({key: kv[1].lstrip()})
except subprocess.CalledProcessError as cp:
# ``apt-cache`` returns 100 if none of the packages asked for
# exist in the apt cache.
if cp.returncode != 100:
raise
return pkgs


def init():
"""Compability shim that does nothing."""
pass


def upstream_version(version):
"""Extracts upstream version from a version string.
Upstream reference: https://salsa.debian.org/apt-team/apt/blob/master/
apt-pkg/deb/debversion.cc#L259
:param version: Version string
:type version: str
:returns: Upstream version
:rtype: str
"""
if version:
version = version[version.find(':') + 1:]
if '-' in version:
version = version[:version.find('-')]
return version


def version_compare(a, b):
"""Compare the given versions.
Call out to ``dpkg`` to make sure the code doing the comparison is
compatible with what the ``apt`` library would do. Mimic the return
values.
Upstream reference:
https://apt-team.pages.debian.net/python-apt/library/apt_pkg.html
?highlight=version_compare#apt_pkg.version_compare
:param a: version string
:type a: str
:param b: version string
:type b: str
:returns: >0 if ``a`` is greater than ``b``, 0 if a equals b,
<0 if ``a`` is smaller than ``b``
:rtype: int
:raises: subprocess.CalledProcessError, RuntimeError
"""
for op in ('gt', 1), ('eq', 0), ('lt', -1):
try:
subprocess.check_call(['dpkg', '--compare-versions',
a, op[0], b],
stderr=subprocess.STDOUT,
universal_newlines=True)
return op[1]
except subprocess.CalledProcessError as cp:
if cp.returncode == 1:
continue
raise
else:
raise RuntimeError('Unable to compare "{}" and "{}", according to '
'our logic they are neither greater, equal nor '
'less than each other.')
@@ -35,6 +35,7 @@
'Tempita',
'Jinja2',
'six',
'psutil',
],
'packages': find_packages(exclude=('tests', 'tests.*', 'tools', 'tools.*')),
'scripts': [
@@ -1,7 +1,6 @@
# Test-only dependencies are unpinned.
#
git+https://git.launchpad.net/ubuntu/+source/python-distutils-extra
git+https://git.launchpad.net/python-apt@1.1.y-xenial
pip
six
coverage>=3.6
ProTip! Use n and p to navigate between commits in a pull request.
You can’t perform that action at this time.