feat(hotplug): add cmd to enable hotplug (#4821)

feat(hotplug): add cmd to enable hotplug
    
Add command to enable network config updates on hotplug events on
already booted instances:
    
cloud-init devel hotplug-hook -s net enable
    
The command creates a sentinel file in
/var/lib/cloud/hotplug.enabled containing the subsystems for 
which is enabled, which will override the allowed events, and
install the appropriate udev rules.

cloudinit/cmd/devel/hotplug_hook.py

@@ -4,12 +4,14 @@
 """Handle reconfiguration on hotplug events."""
 import abc
 import argparse
+import json
 import logging
 import os
 import sys
 import time
 
-from cloudinit import log, reporting, stages
+from cloudinit import log, reporting, settings, stages, util
+from cloudinit.config.cc_install_hotplug import install_hotplug
 from cloudinit.event import EventScope, EventType
 from cloudinit.net import read_sys_net_safe
 from cloudinit.net.network_state import parse_net_config_data
@@ -68,6 +70,10 @@ def get_parser(parser=None):
         choices=["add", "remove"],
     )
 
+    subparsers.add_parser(
+        "enable", help="Enable hotplug for a given subsystem."
+    )
+
     return parser
 
 
@@ -237,6 +243,41 @@ def handle_hotplug(hotplug_init: Init, devpath, subsystem, udevaction):
         raise last_exception
 
 
+def enable_hotplug(hotplug_init: Init, subsystem) -> bool:
+    datasource = hotplug_init.fetch(existing="trust")
+    if not datasource:
+        return False
+    scope = SUBSYSTEM_PROPERTIES_MAP[subsystem][1]
+    hotplug_supported = EventType.HOTPLUG in (
+        datasource.get_supported_events([EventType.HOTPLUG]).get(scope, set())
+    )
+    if not hotplug_supported:
+        print(
+            f"hotplug not supported for event of {subsystem}", file=sys.stderr
+        )
+        return False
+    hotplug_enabled_file = util.read_hotplug_enabled_file()
+    if scope.value in hotplug_enabled_file["scopes"]:
+        print(
+            f"Not installing hotplug for event of type {subsystem}."
+            " Reason: Already done.",
+            file=sys.stderr,
+        )
+        return True
+
+    hotplug_enabled_file["scopes"].append(scope.value)
+    util.write_file(
+        settings.HOTPLUG_ENABLED_FILE,
+        json.dumps(hotplug_enabled_file),
+        omode="w",
+        mode=0o640,
+    )
+    install_hotplug(
+        datasource, network_hotplug_enabled=True, cfg=hotplug_init.cfg
+    )
+    return True
+
+
 def handle_args(name, args):
     # Note that if an exception happens between now and when logging is
     # setup, we'll only see it in the journal
@@ -275,13 +316,29 @@ def handle_args(name, args):
                     )
                     sys.exit(1)
                 print("enabled" if datasource else "disabled")
-            else:
+            elif args.hotplug_action == "handle":
                 handle_hotplug(
                     hotplug_init=hotplug_init,
                     devpath=args.devpath,
                     subsystem=args.subsystem,
                     udevaction=args.udevaction,
                 )
+            else:
+                if os.getuid() != 0:
+                    sys.stderr.write(
+                        "Root is required. Try prepending your command with"
+                        " sudo.\n"
+                    )
+                    sys.exit(1)
+                if not enable_hotplug(
+                    hotplug_init=hotplug_init, subsystem=args.subsystem
+                ):
+                    sys.exit(1)
+                print(
+                    f"Enabled cloud-init hotplug for "
+                    f"subsystem={args.subsystem}"
+                )
+
         except Exception:
             LOG.exception("Received fatal exception handling hotplug!")
             raise

cloudinit/config/cc_install_hotplug.py

@@ -11,6 +11,7 @@
 from cloudinit.distros import ALL_DISTROS
 from cloudinit.event import EventScope, EventType
 from cloudinit.settings import PER_INSTANCE
+from cloudinit.sources import DataSource
 
 meta: MetaSchema = {
     "id": "cc_install_hotplug",
@@ -71,20 +72,18 @@
 """
 
 
-def handle(name: str, cfg: Config, cloud: Cloud, args: list) -> None:
-    network_hotplug_enabled = (
-        "updates" in cfg
-        and "network" in cfg["updates"]
-        and "when" in cfg["updates"]["network"]
-        and "hotplug" in cfg["updates"]["network"]["when"]
-    )
+def install_hotplug(
+    datasource: DataSource,
+    cfg: Config,
+    network_hotplug_enabled: bool,
+):
     hotplug_supported = EventType.HOTPLUG in (
-        cloud.datasource.get_supported_events([EventType.HOTPLUG]).get(
+        datasource.get_supported_events([EventType.HOTPLUG]).get(
             EventScope.NETWORK, set()
         )
     )
     hotplug_enabled = stages.update_event_enabled(
-        datasource=cloud.datasource,
+        datasource=datasource,
         cfg=cfg,
         event_source_type=EventType.HOTPLUG,
         scope=EventScope.NETWORK,
@@ -107,8 +106,8 @@ def handle(name: str, cfg: Config, cloud: Cloud, args: list) -> None:
         return
 
     extra_rules = (
-        cloud.datasource.extra_hotplug_udev_rules
-        if cloud.datasource.extra_hotplug_udev_rules is not None
+        datasource.extra_hotplug_udev_rules
+        if datasource.extra_hotplug_udev_rules is not None
         else ""
     )
     if extra_rules:
@@ -117,10 +116,21 @@ def handle(name: str, cfg: Config, cloud: Cloud, args: list) -> None:
     libexecdir = "/usr/libexec/cloud-init"
     if not os.path.exists(libexecdir):
         libexecdir = "/usr/lib/cloud-init"
+    LOG.info("Installing hotplug.")
     util.write_file(
         filename=HOTPLUG_UDEV_PATH,
         content=HOTPLUG_UDEV_RULES_TEMPLATE.format(
             extra_rules=extra_rules, libexecdir=libexecdir
         ),
     )
     subp.subp(["udevadm", "control", "--reload-rules"])
+
+
+def handle(name: str, cfg: Config, cloud: Cloud, args: list) -> None:
+    network_hotplug_enabled = (
+        "updates" in cfg
+        and "network" in cfg["updates"]
+        and "when" in cfg["updates"]["network"]
+        and "hotplug" in cfg["updates"]["network"]["when"]
+    )
+    install_hotplug(cloud.datasource, cfg, network_hotplug_enabled)

cloudinit/settings.py

@@ -75,3 +75,5 @@
 
 # Used to sanity check incoming handlers/modules frequencies
 FREQUENCIES = [PER_INSTANCE, PER_ALWAYS, PER_ONCE]
+
+HOTPLUG_ENABLED_FILE = "/var/lib/cloud/hotplug.enabled"

cloudinit/stages.py

@@ -40,6 +40,7 @@
 from cloudinit.reporting import events
 from cloudinit.settings import (
     CLOUD_CONFIG,
+    HOTPLUG_ENABLED_FILE,
     PER_ALWAYS,
     PER_INSTANCE,
     PER_ONCE,
@@ -91,6 +92,24 @@ def update_event_enabled(
             copy.deepcopy(default_events),
         ]
     )
+
+    # Add supplemental hotplug event if supported and present in
+    # settings.HOTPLUG_ENABLED_FILE
+    if EventType.HOTPLUG in datasource.supported_update_events.get(
+        scope, set()
+    ):
+        hotplug_enabled_file = util.read_hotplug_enabled_file()
+        if scope.value in hotplug_enabled_file["scopes"]:
+            LOG.debug(
+                "Adding event: scope=%s EventType=%s found in %s",
+                scope,
+                EventType.HOTPLUG,
+                HOTPLUG_ENABLED_FILE,
+            )
+            if not allowed.get(scope):
+                allowed[scope] = set()
+            allowed[scope].add(EventType.HOTPLUG)
+
     LOG.debug("Allowed events: %s", allowed)
 
     scopes: Iterable[EventScope] = [scope]
@@ -335,7 +354,6 @@ def _get_data_source(self, existing) -> sources.DataSource:
             description="attempting to read from cache [%s]" % existing,
             parent=self.reporter,
         ) as myrep:
-
             ds, desc = self._restore_from_checked_cache(existing)
             myrep.description = desc
             self.ds_restored = bool(ds)
@@ -627,7 +645,7 @@ def register_handlers_in_dir(path):
             if not path or not os.path.isdir(path):
                 return
             potential_handlers = util.get_modules_from_dir(path)
-            for (fname, mod_name) in potential_handlers.items():
+            for fname, mod_name in potential_handlers.items():
                 try:
                     mod_locs, looked_locs = importer.find_module(
                         mod_name, [""], ["list_types", "handle_part"]
@@ -675,7 +693,7 @@ def register_handlers_in_dir(path):
 
         def init_handlers():
             # Init the handlers first
-            for (_ctype, mod) in c_handlers.items():
+            for _ctype, mod in c_handlers.items():
                 if mod in c_handlers.initialized:
                     # Avoid initiating the same module twice (if said module
                     # is registered to more than one content-type).
@@ -702,7 +720,7 @@ def walk_handlers(excluded):
 
         def finalize_handlers():
             # Give callbacks opportunity to finalize
-            for (_ctype, mod) in c_handlers.items():
+            for _ctype, mod in c_handlers.items():
                 if mod not in c_handlers.initialized:
                     # Said module was never inited in the first place, so lets
                     # not attempt to finalize those that never got called.

cloudinit/util.py

@@ -59,6 +59,7 @@
     mergers,
     net,
     safeyaml,
+    settings,
     subp,
     temp_utils,
     type_utils,
@@ -3287,3 +3288,23 @@ def decorator(*args, **kwargs):
         return decorator
 
     return wrapper
+
+
+def read_hotplug_enabled_file() -> dict:
+    content: dict = {"scopes": []}
+    try:
+        content = json.loads(
+            load_text_file(settings.HOTPLUG_ENABLED_FILE, quiet=False)
+        )
+    except FileNotFoundError:
+        LOG.debug("File not found: %s", settings.HOTPLUG_ENABLED_FILE)
+    except json.JSONDecodeError as e:
+        LOG.warning(
+            "Ignoring contents of %s because it is not decodable. Error: %s",
+            settings.HOTPLUG_ENABLED_FILE,
+            e,
+        )
+    else:
+        if "scopes" not in content:
+            content["scopes"] = []
+    return content

doc/rtd/reference/cli.rst

@@ -160,11 +160,30 @@ content with any :file:`instance-data.json` variables present.
 :command:`hotplug-hook`
 -----------------------
 
-Respond to newly added system devices by retrieving updated system metadata
-and bringing up/down the corresponding device. This command is intended to be
+Hotplug related subcommands. This command is intended to be
 called via a ``systemd`` service and is not considered user-accessible except
 for debugging purposes.
 
+
+:command:`query`
+^^^^^^^^^^^^^^^^
+
+Query if hotplug is enabled for a given subsystem.
+
+:command:`handle`
+^^^^^^^^^^^^^^^^^
+
+Respond to newly added system devices by retrieving updated system metadata
+and bringing up/down the corresponding device.
+
+:command:`enable`
+^^^^^^^^^^^^^^^^^
+
+Enable hotplug for a given subsystem. This is a last resort command for
+administrators to enable hotplug in running instances. The recommended
+method is configuring :ref:`events`, if not enabled by default in the active
+datasource.
+
 .. _cli_features:
 
 :command:`features`