feat(ec2): support multi NIC/IP setups

cloudinit/config/cc_install_hotplug.py

@@ -21,7 +21,7 @@
         This module will install the udev rules to enable hotplug if
         supported by the datasource and enabled in the userdata. The udev
         rules will be installed as
-        ``/etc/udev/rules.d/10-cloud-init-hook-hotplug.rules``.
+        ``/etc/udev/rules.d/90-cloud-init-hook-hotplug.rules``.
 
         When hotplug is enabled, newly added network devices will be added
         to the system by cloud-init. After udev detects the event,
@@ -59,10 +59,12 @@
 LOG = logging.getLogger(__name__)
 
 
-HOTPLUG_UDEV_PATH = "/etc/udev/rules.d/10-cloud-init-hook-hotplug.rules"
+# 90 to be sorted after 80-net-setup-link.rules which sets ID_NET_DRIVER and
+# some datasources match on drivers
+HOTPLUG_UDEV_PATH = "/etc/udev/rules.d/90-cloud-init-hook-hotplug.rules"
 HOTPLUG_UDEV_RULES_TEMPLATE = """\
 # Installed by cloud-init due to network hotplug userdata
-ACTION!="add|remove", GOTO="cloudinit_end"
+ACTION!="add|remove", GOTO="cloudinit_end"{extra_rules}
 LABEL="cloudinit_hook"
 SUBSYSTEM=="net", RUN+="{libexecdir}/hook-hotplug"
 LABEL="cloudinit_end"
@@ -104,12 +106,21 @@ def handle(name: str, cfg: Config, cloud: Cloud, args: list) -> None:
         LOG.debug("Skipping hotplug install, udevadm not found")
         return
 
+    extra_rules = (
+        cloud.datasource.extra_hotplug_udev_rules
+        if cloud.datasource.extra_hotplug_udev_rules is not None
+        else ""
+    )
+    if extra_rules:
+        extra_rules = "\n" + extra_rules
     # This may need to turn into a distro property at some point
     libexecdir = "/usr/libexec/cloud-init"
     if not os.path.exists(libexecdir):
         libexecdir = "/usr/lib/cloud-init"
     util.write_file(
         filename=HOTPLUG_UDEV_PATH,
-        content=HOTPLUG_UDEV_RULES_TEMPLATE.format(libexecdir=libexecdir),
+        content=HOTPLUG_UDEV_RULES_TEMPLATE.format(
+            extra_rules=extra_rules, libexecdir=libexecdir
+        ),
     )
     subp.subp(["udevadm", "control", "--reload-rules"])

cloudinit/sources/DataSourceEc2.py

@@ -12,12 +12,13 @@
 import logging
 import os
 import time
-from typing import List
+from typing import Dict, List
 
 from cloudinit import dmi, net, sources
 from cloudinit import url_helper as uhelp
 from cloudinit import util, warnings
 from cloudinit.event import EventScope, EventType
+from cloudinit.net import activators
 from cloudinit.net.dhcp import NoDHCPLeaseError
 from cloudinit.net.ephemeral import EphemeralIPNetwork
 from cloudinit.sources.helpers import ec2
@@ -53,9 +54,15 @@ def skip_404_tag_errors(exception):
 # Cloud platforms that support IMDSv2 style metadata server
 IDMSV2_SUPPORTED_CLOUD_PLATFORMS = [CloudNames.AWS, CloudNames.ALIYUN]
 
+# Only trigger hook-hotplug on NICs with Ec2 drivers. Avoid triggering
+# it on docker virtual NICs and the like. LP: #1946003
+_EXTRA_HOTPLUG_UDEV_RULES = """
+ENV{ID_NET_DRIVER}=="vif|ena|ixgbevf", GOTO="cloudinit_hook"
+GOTO="cloudinit_end"
+"""
 
-class DataSourceEc2(sources.DataSource):
 
+class DataSourceEc2(sources.DataSource):
     dsname = "Ec2"
     # Default metadata urls that will be used if none are provided
     # They will be checked for 'resolveability' and some of the
@@ -97,10 +104,23 @@ class DataSourceEc2(sources.DataSource):
         }
     }
 
+    default_update_events = {
+        EventScope.NETWORK: {
+            EventType.BOOT_NEW_INSTANCE,
+            EventType.HOTPLUG,
+        }
+    }
+
+    extra_hotplug_udev_rules = _EXTRA_HOTPLUG_UDEV_RULES
+
     def __init__(self, sys_cfg, distro, paths):
         super(DataSourceEc2, self).__init__(sys_cfg, distro, paths)
         self.metadata_address = None
 
+    def _unpickle(self, ci_pkl_version: int) -> None:
+        super()._unpickle(ci_pkl_version)
+        self.extra_hotplug_udev_rules = _EXTRA_HOTPLUG_UDEV_RULES
+
     def _get_cloud_name(self):
         """Return the cloud name as identified during _get_data."""
         return identify_platform()
@@ -402,7 +422,7 @@ def device_name_to_device(self, name):
             LOG.debug("block-device-mapping not a dictionary: '%s'", bdm)
             return None
 
-        for (entname, device) in bdm.items():
+        for entname, device in bdm.items():
             if entname == name:
                 found = device
                 break
@@ -508,6 +528,7 @@ def network_config(self):
             # behavior on those releases.
             result = convert_ec2_metadata_network_config(
                 net_md,
+                self.distro,
                 fallback_nic=iface,
                 full_network_config=util.get_cfg_option_bool(
                     self.ds_cfg, "apply_full_imds_network_config", True
@@ -871,15 +892,77 @@ def _collect_platform_data():
     return data
 
 
+def _build_nic_order(
+    macs_metadata: Dict[str, Dict], macs: List[str]
+) -> Dict[str, int]:
+    """
+    Builds a dictionary containing macs as keys nad nic orders as values,
+    taking into account `network-card` and `device-number` if present.
+
+    Note that the first NIC will be the primary NIC as it will be the one with
+    [network-card] == 0 and device-number == 0 if present.
+
+    @param macs_metadata: dictionary with mac address as key and contents like:
+    {"device-number": "0", "interface-id": "...", "local-ipv4s": ...}
+    @macs: list of macs to consider
+
+    @return: Dictionary with macs as keys and nic orders as values.
+    """
+    nic_order: Dict[str, int] = {}
+    if len(macs) == 0 or len(macs_metadata) == 0:
+        return nic_order
+
+    valid_macs_metadata = filter(
+        # filter out nics without metadata (not a physical nic)
+        lambda mmd: mmd[1] is not None,
+        # filter by macs
+        map(lambda mac: (mac, macs_metadata.get(mac)), macs),
+    )
+
+    def _get_key_as_int_or(dikt, key, alt_value):
+        value = dikt.get(key, None)
+        if value is not None:
+            return int(value)
+        return alt_value
+
+    # Sort by (network_card, device_index) as some instances could have
+    # multiple network cards with repeated device indexes.
+    #
+    # On platforms where network-card and device-number are not present,
+    # as AliYun, the order will be by mac, as before the introduction of this
+    # function.
+    return {
+        mac: i
+        for i, (mac, _mac_metadata) in enumerate(
+            sorted(
+                valid_macs_metadata,
+                key=lambda mmd: (
+                    _get_key_as_int_or(
+                        mmd[1], "network-card", float("infinity")
+                    ),
+                    _get_key_as_int_or(
+                        mmd[1], "device-number", float("infinity")
+                    ),
+                ),
+            )
+        )
+    }
+
+
 def convert_ec2_metadata_network_config(
-    network_md, macs_to_nics=None, fallback_nic=None, full_network_config=True
+    network_md,
+    distro,
+    macs_to_nics=None,
+    fallback_nic=None,
+    full_network_config=True,
 ):
     """Convert ec2 metadata to network config version 2 data dict.
 
     @param: network_md: 'network' portion of EC2 metadata.
        generally formed as {"interfaces": {"macs": {}} where
        'macs' is a dictionary with mac address as key and contents like:
        {"device-number": "0", "interface-id": "...", "local-ipv4s": ...}
+    @param: distro: instance of Distro.
     @param: macs_to_nics: Optional dict of mac addresses and nic names. If
        not provided, get_interfaces_by_mac is called to get it from the OS.
     @param: fallback_nic: Optionally provide the primary nic interface name.
@@ -913,34 +996,137 @@ def convert_ec2_metadata_network_config(
         netcfg["ethernets"][nic_name] = dev_config
         return netcfg
     # Apply network config for all nics and any secondary IPv4/v6 addresses
-    nic_idx = 0
-    for mac, nic_name in sorted(macs_to_nics.items()):
+    is_netplan = distro.network_activator == activators.NetplanActivator
+    macs = sorted(macs_to_nics.keys())
+    nic_order = _build_nic_order(macs_metadata, macs)
+    for mac in macs:
+        nic_name = macs_to_nics[mac]
         nic_metadata = macs_metadata.get(mac)
         if not nic_metadata:
             continue  # Not a physical nic represented in metadata
-        # device-number is zero-indexed, we want it 1-indexed for the
-        # multiplication on the following line
-        nic_idx = int(nic_metadata.get("device-number", nic_idx)) + 1
-        dhcp_override = {"route-metric": nic_idx * 100}
+        nic_idx = nic_order[mac]
+        is_primary_nic = nic_idx == 0
+        # nic_idx + 1 to start route_metric at 100 (nic_idx is 0-indexed)
+        dhcp_override = {"route-metric": (nic_idx + 1) * 100}
         dev_config = {
             "dhcp4": True,
             "dhcp4-overrides": dhcp_override,
             "dhcp6": False,
             "match": {"macaddress": mac.lower()},
             "set-name": nic_name,
         }
+        # Configure policy-based routing on secondary NICs / secondary IPs to
+        # ensure outgoing packets are routed via the correct interface.
+        #
+        # This config only works on systems using Netplan because Networking
+        # config V2 does not support `routing-policy`, but this config is
+        # passed through on systems using Netplan.
+        #
+        # If device-number is not present (AliYun or other ec2-like platforms),
+        # do not configure source-routing as we cannot determine which is the
+        # primary NIC.
+        if (
+            is_netplan
+            and nic_metadata.get("device-number")
+            and not is_primary_nic
+        ):
+            dhcp_override["use-routes"] = True
+            table = 100 + nic_idx
+            dev_config["routes"] = []
+            try:
+                lease = distro.dhcp_client.dhcp_discovery(
+                    nic_name, distro=distro
+                )
+                gateway = lease["routers"]
+            except NoDHCPLeaseError as e:
+                LOG.warning(
+                    "Could not perform dhcp discovery on %s to find its "
+                    "gateway. Not adding default route via the gateway. "
+                    "Error: %s",
+                    nic_name,
+                    e,
+                )
+            else:
+                # Add default route via the NIC's gateway
+                dev_config["routes"].append(
+                    {
+                        "to": "0.0.0.0/0",
+                        "via": gateway,
+                        "table": table,
+                    },
+                )
+            subnet_prefix_routes = nic_metadata["subnet-ipv4-cidr-block"]
+            subnet_prefix_routes = (
+                [subnet_prefix_routes]
+                if isinstance(subnet_prefix_routes, str)
+                else subnet_prefix_routes
+            )
+            for prefix_route in subnet_prefix_routes:
+                dev_config["routes"].append(
+                    {
+                        "to": prefix_route,
+                        "table": table,
+                    },
+                )
+
+            dev_config["routing-policy"] = []
+            # Packets coming from any IPv4 associated with the current NIC
+            # will be routed using `table` routing table
+            ipv4s = nic_metadata["local-ipv4s"]
+            ipv4s = [ipv4s] if isinstance(ipv4s, str) else ipv4s
+            for ipv4 in ipv4s:
+                dev_config["routing-policy"].append(
+                    {
+                        "from": ipv4,
+                        "table": table,
+                    },
+                )
         if nic_metadata.get("ipv6s"):  # Any IPv6 addresses configured
             dev_config["dhcp6"] = True
             dev_config["dhcp6-overrides"] = dhcp_override
+            if (
+                is_netplan
+                and nic_metadata.get("device-number")
+                and not is_primary_nic
+            ):
+                table = 100 + nic_idx
+                subnet_prefix_routes = nic_metadata["subnet-ipv6-cidr-block"]
+                subnet_prefix_routes = (
+                    [subnet_prefix_routes]
+                    if isinstance(subnet_prefix_routes, str)
+                    else subnet_prefix_routes
+                )
+                for prefix_route in subnet_prefix_routes:
+                    dev_config["routes"].append(
+                        {
+                            "to": prefix_route,
+                            "table": table,
+                        },
+                    )
+
+                dev_config["routing-policy"] = []
+                ipv6s = nic_metadata["ipv6s"]
+                ipv6s = [ipv6s] if isinstance(ipv6s, str) else ipv6s
+                for ipv6 in ipv6s:
+                    dev_config["routing-policy"].append(
+                        {
+                            "from": ipv6,
+                            "table": table,
+                        },
+                    )
         dev_config["addresses"] = get_secondary_addresses(nic_metadata, mac)
         if not dev_config["addresses"]:
             dev_config.pop("addresses")  # Since we found none configured
+
         netcfg["ethernets"][nic_name] = dev_config
-    # Remove route-metric dhcp overrides if only one nic configured
+    # Remove route-metric dhcp overrides and routes / routing-policy if only
+    # one nic configured
     if len(netcfg["ethernets"]) == 1:
         for nic_name in netcfg["ethernets"].keys():
             netcfg["ethernets"][nic_name].pop("dhcp4-overrides")
             netcfg["ethernets"][nic_name].pop("dhcp6-overrides", None)
+            netcfg["ethernets"][nic_name].pop("routes", None)
+            netcfg["ethernets"][nic_name].pop("routing-policy", None)
     return netcfg
 
 

cloudinit/sources/__init__.py

@@ -296,6 +296,9 @@ class DataSource(CloudInitPickleMixin, metaclass=abc.ABCMeta):
     # in the updated metadata
     skip_hotplug_detect = False
 
+    # Extra udev rules for cc_install_hotplug
+    extra_hotplug_udev_rules: Optional[str] = None
+
     _ci_pkl_version = 1
 
     def __init__(self, sys_cfg, distro: Distro, paths: Paths, ud_proc=None):
@@ -344,6 +347,8 @@ def _unpickle(self, ci_pkl_version: int) -> None:
                     e,
                 )
                 raise DatasourceUnpickleUserDataError() from e
+        if not hasattr(self, "extra_hotplug_udev_rules"):
+            self.extra_hotplug_udev_rules = None
 
     def __str__(self):
         return type_utils.obj_name(self)

doc/rtd/reference/datasources/ec2.rst

@@ -150,4 +150,11 @@ Notes
    For example: the primary NIC will have a DHCP route-metric of 100,
    the next NIC will have 200.
 
+ * For EC2 instances with multiple NICs, policy-based routing will be
+   configured on secondary NICs / secondary IPs to ensure outgoing packets
+   are routed via the correct interface.
+   This network configuration is only applied on distros using Netplan and
+   at first boot only but it can be configured to be applied on every boot
+   and when NICs are hotplugged, see :ref:`events`.
+
 .. _EC2 tags user guide: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html#work-with-tags-in-IMDS

systemd/cloud-init-hotplugd.service

@@ -1,6 +1,6 @@
 # Paired with cloud-init-hotplugd.socket to read from the FIFO
 # /run/cloud-init/hook-hotplug-cmd which is created during a udev network
-# add or remove event as processed by 10-cloud-init-hook-hotplug.rules.
+# add or remove event as processed by 90-cloud-init-hook-hotplug.rules.
 
 # On start, read args from the FIFO, process and provide structured arguments
 # to `cloud-init devel hotplug-hook` which will setup or teardown network

systemd/cloud-init-hotplugd.socket

@@ -1,6 +1,6 @@
 # cloud-init-hotplugd.socket listens on the FIFO file
 # /run/cloud-init/hook-hotplug-cmd which is created during a udev network
-# add or remove event as processed by 10-cloud-init-hook-hotplug.rules.
+# add or remove event as processed by 90-cloud-init-hook-hotplug.rules.
 
 # Known bug with an enforcing SELinux policy: LP: #1936229
 [Unit]