apt: install software-properties-common when absent but needed (#4441)

When optional user-data contains APT sources configuration that
warrants APT repo setup, cloud-init calls add-apt-repository which
is packaged in software-properties common.

The software-properties-common package is defined as a Recommends:
in debian/control, and some minimal images do not install
recommended packages.

When minimal images do not have software-properties-common
installed, cloud-init will install this package on first boot only
when required by optional apt sources user-data.

The gnupg package is another optional/recommended package that
willl be installed is specific apt user-data config requires gpg
interaction.

Refactor _ensure_gpg to _ensure_dependencies which now inspects
cloud-config to determine if either gpg or add-apt-repository
are required commands based on optional user-data.
Attempt to install any missing package dependencies before processing
the cloud-config.

cloudinit/config/cc_apt_configure.py

@@ -8,7 +8,6 @@
 
 """Apt Configure: Configure apt for the user."""
 
-import functools
 import glob
 import logging
 import os
@@ -36,6 +35,11 @@
 frequency = PER_INSTANCE
 distros = ["ubuntu", "debian"]
 
+PACKAGE_DEPENDENCY_BY_COMMAND = {
+    "add-apt-repository": "software-properties-common",
+    "gpg": "gnupg",
+}
+
 meta: MetaSchema = {
     "id": "cc_apt_configure",
     "name": "Apt Configure",
@@ -216,6 +220,12 @@ def apply_apt(cfg, cloud, target):
     mirrors = find_apt_mirror_info(cfg, cloud, arch=arch)
     LOG.debug("Apt Mirror info: %s", mirrors)
 
+    matcher = None
+    matchcfg = cfg.get("add_apt_repo_match", ADD_APT_REPO_MATCH)
+    if matchcfg:
+        matcher = re.compile(matchcfg).search
+    _ensure_dependencies(cfg, matcher, cloud)
+
     if util.is_false(cfg.get("preserve_sources_list", False)):
         add_mirror_keys(cfg, cloud, target)
         generate_sources_list(cfg, release, mirrors, cloud)
@@ -232,11 +242,6 @@ def apply_apt(cfg, cloud, target):
         params["RELEASE"] = release
         params["MIRROR"] = mirrors["MIRROR"]
 
-        matcher = None
-        matchcfg = cfg.get("add_apt_repo_match", ADD_APT_REPO_MATCH)
-        if matchcfg:
-            matcher = re.compile(matchcfg).search
-
         add_apt_sources(
             cfg["sources"],
             cloud,
@@ -500,10 +505,37 @@ def add_apt_key_raw(key, file_name, hardened=False, target=None):
         raise
 
 
-@functools.lru_cache(maxsize=1)
-def _ensure_gpg(cloud):
-    if not shutil.which("gpg"):
-        cloud.distro.install_packages(["gnupg"])
+def _ensure_dependencies(cfg, aa_repo_match, cloud):
+    """Install missing package dependencies based on apt_sources config.
+
+    Inspect the cloud config user-data provided. When user-data indicates
+    conditions where add_apt_key or add-apt-repository will be called,
+    ensure the required command dependencies are present installed.
+
+    Perform this inspection upfront because it is very expensive to call
+    distro.install_packages due to a preliminary 'apt update' called before
+    package installation.
+    """
+    missing_packages = []
+    required_cmds = set()
+    if util.is_false(cfg.get("preserve_sources_list", False)):
+        for mirror_key in ("primary", "security"):
+            if cfg.get(mirror_key):
+                # Include gpg when mirror_key non-empty list and any item
+                # defines key or keyid.
+                for mirror_item in cfg[mirror_key]:
+                    if {"key", "keyid"}.intersection(mirror_item):
+                        required_cmds.add("gpg")
+    apt_sources_dict = cfg.get("sources", {})
+    for ent in apt_sources_dict.values():
+        if {"key", "keyid"}.intersection(ent):
+            required_cmds.add("gpg")
+        if aa_repo_match(ent.get("source", "")):
+            required_cmds.add("add-apt-repository")
+    for command in required_cmds:
+        if not shutil.which(command):
+            missing_packages.append(PACKAGE_DEPENDENCY_BY_COMMAND[command])
+    cloud.distro.install_packages(sorted(missing_packages))
 
 
 def add_apt_key(ent, cloud, target=None, hardened=False, file_name=None):
@@ -512,7 +544,6 @@ def add_apt_key(ent, cloud, target=None, hardened=False, file_name=None):
     Supports raw keys or keyid's
     The latter will as a first step fetched to get the raw key
     """
-    _ensure_gpg(cloud)
     if "keyid" in ent and "key" not in ent:
         keyserver = DEFAULT_KEYSERVER
         if "keyserver" in ent:
@@ -580,7 +611,7 @@ def add_apt_sources(
             key_file = add_apt_key(ent, cloud, target, hardened=True)
             template_params["KEY_FILE"] = key_file
         else:
-            key_file = add_apt_key(ent, cloud, target)
+            add_apt_key(ent, cloud, target)
 
         if "source" not in ent:
             continue

tests/integration_tests/modules/test_apt.py

@@ -8,6 +8,7 @@
 from tests.integration_tests.instances import IntegrationInstance
 from tests.integration_tests.integration_settings import PLATFORM
 from tests.integration_tests.releases import CURRENT_RELEASE, IS_UBUNTU
+from tests.integration_tests.util import verify_clean_log
 
 USER_DATA = """\
 #cloud-config
@@ -412,3 +413,28 @@ def test_apt_proxy(client: IntegrationInstance):
     assert 'Acquire::http::Proxy "http://squid.internal:3128";' in out
     assert 'Acquire::ftp::Proxy "ftp://squid.internal:3128";' in out
     assert 'Acquire::https::Proxy "https://squid.internal:3128";' in out
+
+
+INSTALL_ANY_MISSING_RECOMMENDED_DEPENDENCIES = """\
+#cloud-config
+bootcmd:
+    - apt-get remove gpg -y
+apt:
+  sources:
+    test_keyserver:
+      keyid: 110E21D8B0E2A1F0243AF6820856F197B892ACEA
+      keyserver: keyserver.ubuntu.com
+      source: "deb http://ppa.launchpad.net/canonical-kernel-team/ppa/ubuntu $RELEASE main"
+    test_ppa:
+      keyid: 441614D8
+      keyserver: keyserver.ubuntu.com
+      source: "ppa:simplestreams-dev/trunk"
+"""  # noqa: E501
+
+
+@pytest.mark.skipif(not IS_UBUNTU, reason="Apt usage")
+@pytest.mark.user_data(INSTALL_ANY_MISSING_RECOMMENDED_DEPENDENCIES)
+def test_install_missing_deps(client: IntegrationInstance):
+    log = client.read_from_file("/var/log/cloud-init.log")
+    verify_clean_log(log)
+    assert "install gnupg software-properties-common" in log

tests/unittests/config/test_apt_configure_sources_list_v1.py

@@ -71,7 +71,7 @@ def setup(self, mocker):
         self.subp = mocker.patch.object(
             subp, "subp", return_value=("PPID   PID", "")
         )
-        mocker.patch("cloudinit.config.cc_apt_configure._ensure_gpg")
+        mocker.patch("cloudinit.config.cc_apt_configure._ensure_dependencies")
         lsb = mocker.patch("cloudinit.util.lsb_release")
         lsb.return_value = {"codename": "fakerelease"}
         m_arch = mocker.patch("cloudinit.util.get_dpkg_architecture")

tests/unittests/config/test_apt_configure_sources_list_v3.py

@@ -88,7 +88,7 @@ def setup(self, mocker):
         lsb.return_value = {"codename": "fakerel"}
         m_arch = mocker.patch("cloudinit.util.get_dpkg_architecture")
         m_arch.return_value = "amd64"
-        mocker.patch("cloudinit.config.cc_apt_configure._ensure_gpg")
+        mocker.patch("cloudinit.config.cc_apt_configure._ensure_dependencies")
 
     @pytest.mark.parametrize(
         "distro,template_present",

tests/unittests/config/test_apt_source_v1.py

@@ -77,7 +77,7 @@ def common_mocks(self, mocker):
             "cloudinit.util.get_dpkg_architecture", return_value="amd64"
         )
         mocker.patch.object(subp, "subp", return_value=("PPID   PID", ""))
-        mocker.patch("cloudinit.config.cc_apt_configure._ensure_gpg")
+        mocker.patch("cloudinit.config.cc_apt_configure._ensure_dependencies")
 
     def _get_default_params(self):
         """get_default_params

tests/unittests/config/test_apt_source_v3.py

@@ -55,7 +55,7 @@ def setup(self, mocker, tmpdir):
             f"{M_PATH}util.lsb_release",
             return_value=MOCK_LSB_RELEASE_DATA.copy(),
         )
-        mocker.patch(f"{M_PATH}_ensure_gpg")
+        mocker.patch(f"{M_PATH}_ensure_dependencies")
         self.aptlistfile = tmpdir.join("src1.list").strpath
         self.aptlistfile2 = tmpdir.join("src2.list").strpath
         self.aptlistfile3 = tmpdir.join("src3.list").strpath