Permalink
Browse files

Apt: add new apt configuration format

This adds an improved apt configuration format that is fully backwards
compatible with previous behavior. This is mostly copied from curtin's
implementation.

It does:
 * clean up and centralizes many of the top level 'apt_*' values that
   previously existed into a single top level 'apt'key.
 * support a 'source' in apt/sources/entry that has only a key
 * documents new features and adds tests.

See the added doc/examples/cloud-config-apt.txt for more information.
  • Loading branch information...
1 parent 648dbbf commit d861415ff9ab816b1183b8c58ec35348be4fd458 @cpaelzer cpaelzer committed with smoser Aug 10, 2016

Large diffs are not rendered by default.

Oops, something went wrong.
View
@@ -36,11 +36,11 @@ def export_armour(key):
return armour
-def receive_key(key, keyserver):
+def recv_key(key, keyserver):
"""Receive gpg key from the specified keyserver"""
LOG.debug('Receive gpg key "%s"', key)
try:
- util.subp(["gpg", "--keyserver", keyserver, "--recv-keys", key],
+ util.subp(["gpg", "--keyserver", keyserver, "--recv", key],
capture=True)
except util.ProcessExecutionError as error:
raise ValueError(('Failed to import key "%s" '
@@ -57,12 +57,12 @@ def delete_key(key):
LOG.warn('Failed delete key "%s": %s', key, error)
-def get_key_by_id(keyid, keyserver="keyserver.ubuntu.com"):
+def getkeybyid(keyid, keyserver='keyserver.ubuntu.com'):
"""get gpg keyid from keyserver"""
armour = export_armour(keyid)
if not armour:
try:
- receive_key(keyid, keyserver=keyserver)
+ recv_key(keyid, keyserver=keyserver)
armour = export_armour(keyid)
except ValueError:
LOG.exception('Failed to obtain gpg key %s', keyid)
View
@@ -61,6 +61,10 @@
from cloudinit.settings import (CFG_BUILTIN)
+try:
+ string_types = (basestring,)
+except NameError:
+ string_types = (str,)
_DNS_REDIRECT_IP = None
LOG = logging.getLogger(__name__)
@@ -82,6 +86,71 @@
PROC_CMDLINE = None
+_LSB_RELEASE = {}
+
+
+def get_architecture(target=None):
+ out, _ = subp(['dpkg', '--print-architecture'], capture=True,
+ target=target)
+ return out.strip()
+
+
+def _lsb_release(target=None):
+ fmap = {'Codename': 'codename', 'Description': 'description',
+ 'Distributor ID': 'id', 'Release': 'release'}
+
+ data = {}
+ try:
+ out, _ = subp(['lsb_release', '--all'], capture=True, target=target)
+ for line in out.splitlines():
+ fname, _, val = line.partition(":")
+ if fname in fmap:
+ data[fmap[fname]] = val.strip()
+ missing = [k for k in fmap.values() if k not in data]
+ if len(missing):
+ LOG.warn("Missing fields in lsb_release --all output: %s",
+ ','.join(missing))
+
+ except ProcessExecutionError as err:
+ LOG.warn("Unable to get lsb_release --all: %s", err)
+ data = {v: "UNAVAILABLE" for v in fmap.values()}
+
+ return data
+
+
+def lsb_release(target=None):
+ if target_path(target) != "/":
+ # do not use or update cache if target is provided
+ return _lsb_release(target)
+
+ global _LSB_RELEASE
+ if not _LSB_RELEASE:
+ data = _lsb_release()
+ _LSB_RELEASE.update(data)
+ return _LSB_RELEASE
+
+
+def target_path(target, path=None):
+ # return 'path' inside target, accepting target as None
+ if target in (None, ""):
+ target = "/"
+ elif not isinstance(target, string_types):
+ raise ValueError("Unexpected input for target: %s" % target)
+ else:
+ target = os.path.abspath(target)
+ # abspath("//") returns "//" specifically for 2 slashes.
+ if target.startswith("//"):
+ target = target[1:]
+
+ if not path:
+ return target
+
+ # os.path.join("/etc", "/foo") returns "/foo". Chomp all leading /.
+ while len(path) and path[0] == "/":
+ path = path[1:]
+
+ return os.path.join(target, path)
+
def decode_binary(blob, encoding='utf-8'):
# Converts a binary type into a text type using given encoding.
@@ -1688,10 +1757,20 @@ def delete_dir_contents(dirname):
def subp(args, data=None, rcs=None, env=None, capture=True, shell=False,
- logstring=False):
+ logstring=False, decode="replace", target=None):
+
+ # not supported in cloud-init (yet), for now kept in the call signature
+ # to ease maintaining code shared between cloud-init and curtin
+ if target is not None:
+ raise ValueError("target arg not supported by cloud-init")
+
if rcs is None:
rcs = [0]
+
+ devnull_fp = None
try:
+ if target_path(target) != "/":
+ args = ['chroot', target] + list(args)
if not logstring:
LOG.debug(("Running command %s with allowed return codes %s"
@@ -1700,33 +1779,52 @@ def subp(args, data=None, rcs=None, env=None, capture=True, shell=False,
LOG.debug(("Running hidden command to protect sensitive "
"input/output logstring: %s"), logstring)
- if not capture:
- stdout = None
- stderr = None
- else:
+ stdin = None
+ stdout = None
+ stderr = None
+ if capture:
stdout = subprocess.PIPE
stderr = subprocess.PIPE
- stdin = subprocess.PIPE
- kws = dict(stdout=stdout, stderr=stderr, stdin=stdin,
- env=env, shell=shell)
- if six.PY3:
- # Use this so subprocess output will be (Python 3) str, not bytes.
- kws['universal_newlines'] = True
- sp = subprocess.Popen(args, **kws)
+ if data is None:
+ # using devnull assures any reads get null, rather
+ # than possibly waiting on input.
+ devnull_fp = open(os.devnull)
+ stdin = devnull_fp
+ else:
+ stdin = subprocess.PIPE
+ if not isinstance(data, bytes):
+ data = data.encode()
+
+ sp = subprocess.Popen(args, stdout=stdout,
+ stderr=stderr, stdin=stdin,
+ env=env, shell=shell)
(out, err) = sp.communicate(data)
+
+ # Just ensure blank instead of none.
+ if not out and capture:
+ out = b''
+ if not err and capture:
+ err = b''
+ if decode:
+ def ldecode(data, m='utf-8'):
+ if not isinstance(data, bytes):
+ return data
+ return data.decode(m, errors=decode)
+
+ out = ldecode(out)
+ err = ldecode(err)
except OSError as e:
raise ProcessExecutionError(cmd=args, reason=e,
errno=e.errno)
+ finally:
+ if devnull_fp:
+ devnull_fp.close()
+
rc = sp.returncode
if rc not in rcs:
raise ProcessExecutionError(stdout=out, stderr=err,
exit_code=rc,
cmd=args)
- # Just ensure blank instead of none?? (iff capturing)
- if not out and capture:
- out = ''
- if not err and capture:
- err = ''
return (out, err)
@@ -2251,3 +2349,18 @@ def message_from_string(string):
if sys.version_info[:2] < (2, 7):
return email.message_from_file(six.StringIO(string))
return email.message_from_string(string)
+
+
+def get_installed_packages(target=None):
+ (out, _) = subp(['dpkg-query', '--list'], target=target, capture=True)
+
+ pkgs_inst = set()
+ for line in out.splitlines():
+ try:
+ (state, pkg, _) = line.split(None, 2)
+ except ValueError:
+ continue
+ if state.startswith("hi") or state.startswith("ii"):
+ pkgs_inst.add(re.sub(":.*", "", pkg))
+
+ return pkgs_inst
@@ -4,18 +4,21 @@
#
# Default: auto select based on cloud metadata
# in ec2, the default is <region>.archive.ubuntu.com
-# apt_mirror:
-# use the provided mirror
-# apt_mirror_search:
-# search the list for the first mirror.
-# this is currently very limited, only verifying that
-# the mirror is dns resolvable or an IP address
+# apt:
+# primary:
+# - arches [default]
+# uri:
+# use the provided mirror
+# search:
+# search the list for the first mirror.
+# this is currently very limited, only verifying that
+# the mirror is dns resolvable or an IP address
#
-# if neither apt_mirror nor apt_mirror search is set (the default)
+# if neither mirror is set (the default)
# then use the mirror provided by the DataSource found.
# In EC2, that means using <region>.ec2.archive.ubuntu.com
-#
-# if no mirror is provided by the DataSource, and 'apt_mirror_search_dns' is
+#
+# if no mirror is provided by the DataSource, but 'search_dns' is
# true, then search for dns names '<distro>-mirror' in each of
# - fqdn of this host per cloud metadata
# - localdomain
@@ -27,8 +30,19 @@
# up and expose them only by creating dns entries.
#
# if none of that is found, then the default distro mirror is used
-apt_mirror: http://us.archive.ubuntu.com/ubuntu/
-apt_mirror_search:
- - http://local-mirror.mydomain
- - http://archive.ubuntu.com
-apt_mirror_search_dns: False
+apt:
+ primary:
+ - arches: [default]
+ uri: http://us.archive.ubuntu.com/ubuntu/
+# or
+apt:
+ primary:
+ - arches: [default]
+ search:
+ - http://local-mirror.mydomain
+ - http://archive.ubuntu.com
+# or
+apt:
+ primary:
+ - arches: [default]
+ search_dns: True
Oops, something went wrong.

0 comments on commit d861415

Please sign in to comment.