Bug 1915885: Support multiple nodes subnets

doc/source/installation/containerized.rst

@@ -80,7 +80,7 @@ script. Below is the list of available variables:
 * ``$KURYR_K8S_POD_SUBNET_ID`` - ``[neutron_defaults]pod_subnet_id``
 * ``$KURYR_K8S_POD_SG`` - ``[neutron_defaults]pod_sg``
 * ``$KURYR_K8S_SERVICE_SUBNET_ID`` - ``[neutron_defaults]service_subnet_id``
-* ``$KURYR_K8S_WORKER_NODES_SUBNET`` - ``[pod_vif_nested]worker_nodes_subnet``
+* ``$KURYR_K8S_WORKER_NODES_SUBNETS`` - ``[pod_vif_nested]worker_nodes_subnets``
 * ``$KURYR_K8S_BINDING_DRIVER`` - ``[binding]driver`` (default:
   ``kuryr.lib.binding.drivers.vlan``)
 * ``$KURYR_K8S_BINDING_IFACE`` - ``[binding]link_iface`` (default: eth0)

doc/source/installation/devstack/nested-macvlan.rst

@@ -28,7 +28,7 @@ nested MACVLAN driver rather than VLAN and trunk ports.
      .. code-block:: ini
 
         [pod_vif_nested]
-        worker_nodes_subnet = <UNDERCLOUD_SUBNET_WORKER_NODES_UUID>
+        worker_nodes_subnets = <UNDERCLOUD_SUBNET_WORKER_NODES_UUID>
 
    - Configure "pod_vif_driver" as "nested-macvlan":
 

doc/source/installation/devstack/nested-vlan.rst

@@ -64,7 +64,7 @@ for the VM:
       .. code-block:: ini
 
          [pod_vif_nested]
-         worker_nodes_subnet = <UNDERCLOUD_SUBNET_WORKER_NODES_UUID>
+         worker_nodes_subnets = <UNDERCLOUD_SUBNET_WORKER_NODES_UUID>
 
    - Configure binding section:
 

doc/source/nested_vlan_mode.rst

@@ -0,0 +1,66 @@
+=================================
+Kuryr-Kubernetes nested VLAN mode
+=================================
+
+Kuryr-Kubernetes can work in two basic modes - nested and standalone. The main
+use case of the project, which is to support Kubernetes running on OpenStack
+VMs is implemented with nested mode. The standalone mode is mostly used for
+testing.
+
+This document describes nested VLAN mode.
+
+
+Requirements
+============
+
+Nested VLAN mode requires Neutron to have `trunk` extension enabled, which adds
+trunk port functionality to Neutron API.
+
+
+Principle
+=========
+
+This mode aims at use case of kuryr-kubernetes providing networking for a
+Kubernetes cluster running in VMs on OpenStack.
+
+.. note::
+
+   A natural consideration here is running kuryr-kubernetes in containers on
+   that K8s cluster. For more see :ref:`containerized` section.
+
+The principle of nested VLAN is that Kuryr-Kubernetes will require that main
+interface of the K8s worker VMs is a trunk port. Then each of the pods will
+get a subport of that attached into its network namespace.
+
+
+How to configure
+================
+
+You need to set several options in the kuryr.conf:
+
+.. code-block:: ini
+
+   [binding]
+   default_driver = kuryr.lib.binding.drivers.vlan
+   # Name of the trunk port interface on VMs. If not provided Kuryr will try
+   # to autodetect it.
+   link_iface = ens3
+
+   [kubernetes]
+   pod_vif_driver = nested-vlan
+   vif_pool_driver = nested  # If using port pools.
+
+   [pod_vif_nested]
+   # ID of the subnet in which worker node VMs are running (if multiple join
+   # with a comma).
+   worker_nodes_subnets = <id>
+
+Also if you want to run several Kubernetes cluster in one OpenStack tenant you
+need to make sure Kuryr-Kubernetes instances are able to distinguish their own
+resources from resources created by other instances. In order to do that you
+need to configure Kuryr-Kubernetes to tag resources with unique ID:
+
+.. code-block:: ini
+
+   [neutron_defaults]
+   resource_tags = <unique-id-of-the-K8s-cluster>

kuryr_kubernetes/config.py

@@ -142,6 +142,9 @@
                help=_("The driver that manages VIFs pools for "
                       "Kubernetes Pods"),
                default='noop'),
+    cfg.StrOpt('nodes_subnets_driver',
+               help=_("The driver that manages listing K8s nodes subnet_ids."),
+               default='config'),
     cfg.BoolOpt('port_debug',
                 help=_('Enable port debug to force kuryr port names to be '
                        'set to their corresponding pod names.'),
@@ -268,9 +271,11 @@
 ]
 
 nested_vif_driver_opts = [
-    cfg.StrOpt('worker_nodes_subnet',
-               help=_("Neutron subnet ID for k8s worker node vms."),
-               default=''),
+    cfg.ListOpt('worker_nodes_subnets',
+                help=_("Neutron subnet IDs for k8s worker node VMs."),
+                default=[],
+                deprecated_name='worker_nodes_subnet',
+                deprecated_group='pod_vif_nested'),
     cfg.IntOpt('rev_update_attempts',
                help=_("How many time to try to re-update the neutron resource "
                       "when revision has been changed by other thread"),

kuryr_kubernetes/constants.py

@@ -43,6 +43,9 @@
 K8S_OBJ_KURYRLOADBALANCER = 'KuryrLoadBalancer'
 K8S_OBJ_KURYRPORT = 'KuryrPort'
 
+OPENSHIFT_OBJ_MACHINE = 'Machine'
+OPENSHIFT_API_CRD_MACHINES = '/apis/machine.openshift.io/v1beta1/machines'
+
 K8S_POD_STATUS_PENDING = 'Pending'
 K8S_POD_STATUS_SUCCEEDED = 'Succeeded'
 K8S_POD_STATUS_FAILED = 'Failed'

kuryr_kubernetes/controller/drivers/base.py

@@ -751,3 +751,35 @@ def get_project(self, policy):
         :returns: OpenStack project_id
         """
         raise NotImplementedError()
+
+
+class NodesSubnetsDriver(DriverBase, metaclass=abc.ABCMeta):
+    """Keeps list of subnet_ids of the OpenShift Nodes."""
+
+    ALIAS = 'nodes_subnets'
+
+    @abc.abstractmethod
+    def get_nodes_subnets(self, raise_on_empty=False):
+        """Gets list of subnet_ids of OpenShift Nodes.
+
+        :param raise_on_empty: whether it should raise if list is empty.
+        :return: list of subnets
+        """
+
+        raise NotImplementedError()
+
+    @abc.abstractmethod
+    def add_node(self, node):
+        """Handles node addition.
+
+        :param node: Node object
+        """
+        pass
+
+    @abc.abstractmethod
+    def delete_node(self, node):
+        """Handles node removal
+
+        :param node: Node object
+        """
+        pass

kuryr_kubernetes/controller/drivers/nested_vif.py

@@ -20,38 +20,46 @@
 from oslo_log import log as logging
 
 from kuryr_kubernetes import clients
+from kuryr_kubernetes.controller.drivers import base
 from kuryr_kubernetes.controller.drivers import neutron_vif
 
 
+CONF = oslo_cfg.CONF
 LOG = logging.getLogger(__name__)
 
 
 class NestedPodVIFDriver(neutron_vif.NeutronPodVIFDriver,
                          metaclass=abc.ABCMeta):
     """Skeletal handler driver for VIFs for Nested Pods."""
 
+    def __init__(self):
+        super().__init__()
+        self.nodes_subnets_driver = base.NodesSubnetsDriver.get_instance()
+
     def _get_parent_port_by_host_ip(self, node_fixed_ip):
         os_net = clients.get_network_client()
-        node_subnet_id = oslo_cfg.CONF.pod_vif_nested.worker_nodes_subnet
-        if not node_subnet_id:
-            raise oslo_cfg.RequiredOptError(
-                'worker_nodes_subnet', oslo_cfg.OptGroup('pod_vif_nested'))
+        node_subnet_ids = self.nodes_subnets_driver.get_nodes_subnets(
+            raise_on_empty=True)
 
+        fixed_ips = ['ip_address=%s' % str(node_fixed_ip)]
+        filters = {'fixed_ips': fixed_ips}
+        tags = CONF.neutron_defaults.resource_tags
+        if tags:
+            filters['tags'] = tags
         try:
-            fixed_ips = ['subnet_id=%s' % str(node_subnet_id),
-                         'ip_address=%s' % str(node_fixed_ip)]
-            ports = os_net.ports(fixed_ips=fixed_ips)
+            ports = os_net.ports(**filters)
         except os_exc.SDKException:
-            LOG.error("Parent vm port with fixed ips %s not found!",
-                      fixed_ips)
+            LOG.error("Parent VM port with fixed IPs %s not found!", fixed_ips)
             raise
 
-        try:
-            return next(ports)
-        except StopIteration:
-            LOG.error("Neutron port for vm port with fixed ips %s not found!",
-                      fixed_ips)
-            raise kl_exc.NoResourceException
+        for port in ports:
+            for fip in port.fixed_ips:
+                if fip.get('subnet_id') in node_subnet_ids:
+                    return port
+
+        LOG.error("Neutron port for VM port with fixed IPs %s not found!",
+                  fixed_ips)
+        raise kl_exc.NoResourceException()
 
     def _get_parent_port(self, pod):
         try:

kuryr_kubernetes/controller/drivers/network_policy.py

@@ -37,6 +37,7 @@ class NetworkPolicyDriver(base.NetworkPolicyDriver):
     def __init__(self):
         self.os_net = clients.get_network_client()
         self.kubernetes = clients.get_kubernetes_client()
+        self.nodes_subnets_driver = base.NodesSubnetsDriver.get_instance()
 
     def ensure_network_policy(self, policy):
         """Create security group rules out of network policies
@@ -147,9 +148,9 @@ def _get_default_np_rules(self):
         if CONF.octavia_defaults.enforce_sg_rules:
             default_cidrs.append(utils.get_subnet_cidr(
                 CONF.neutron_defaults.service_subnet))
-        worker_subnet_id = CONF.pod_vif_nested.worker_nodes_subnet
-        if worker_subnet_id:
-            default_cidrs.append(utils.get_subnet_cidr(worker_subnet_id))
+        worker_subnet_ids = self.nodes_subnets_driver.get_nodes_subnets()
+        default_cidrs.extend(utils.get_subnets_cidrs(worker_subnet_ids))
+
         for cidr in default_cidrs:
             ethertype = constants.IPv4
             if ipaddress.ip_network(cidr).version == constants.IP_VERSION_6:

kuryr_kubernetes/controller/drivers/node_subnets.py

@@ -0,0 +1,125 @@
+# Copyright 2020 Red Hat, Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from oslo_concurrency import lockutils
+from oslo_config import cfg
+from oslo_log import log as logging
+
+from kuryr_kubernetes import clients
+from kuryr_kubernetes import constants
+from kuryr_kubernetes.controller.drivers import base
+from kuryr_kubernetes import exceptions
+from kuryr_kubernetes import utils
+
+
+CONF = cfg.CONF
+LOG = logging.getLogger(__name__)
+
+
+class ConfigNodesSubnets(base.NodesSubnetsDriver):
+    """Provides list of nodes subnets from configuration."""
+
+    def get_nodes_subnets(self, raise_on_empty=False):
+        node_subnet_ids = CONF.pod_vif_nested.worker_nodes_subnets
+        if not node_subnet_ids:
+            if raise_on_empty:
+                raise cfg.RequiredOptError(
+                    'worker_nodes_subnets', cfg.OptGroup('pod_vif_nested'))
+            else:
+                return []
+
+        return node_subnet_ids
+
+    def add_node(self, node):
+        return False
+
+    def delete_node(self, node):
+        return False
+
+
+class OpenShiftNodesSubnets(base.NodesSubnetsDriver):
+    """Provides list of nodes subnets based on OpenShift Machine objects."""
+
+    def __init__(self):
+        super().__init__()
+        self.subnets = set()
+
+    def _get_subnet_from_machine(self, machine):
+        networks = machine['spec'].get(
+            'providerSpec', {}).get('value', {}).get('networks')
+        if not networks:
+            LOG.warning('No `networks` in Machine `providerSpec`')
+            return None
+
+        subnets = networks[0].get('subnets')
+        if not subnets:
+            LOG.warning('No `subnets` in Machine `providerSpec.values.'
+                        'networks`.')
+            return None
+
+        primary_subnet = subnets[0]
+        if primary_subnet.get('uuid'):
+            return primary_subnet['uuid']
+        else:
+            subnet_filter = primary_subnet['filter']
+            subnet_id = utils.get_subnet_id(**subnet_filter)
+            return subnet_id
+
+    def get_nodes_subnets(self, raise_on_empty=False):
+        with lockutils.lock('kuryr-machine-add'):
+            if not self.subnets and raise_on_empty:
+                raise exceptions.ResourceNotReady(
+                    'OpenShift Machines does not exist or are not yet '
+                    'handled. Cannot determine worker nodes subnets.')
+
+            return list(self.subnets)
+
+    def add_node(self, machine):
+        subnet_id = self._get_subnet_from_machine(machine)
+        if not subnet_id:
+            LOG.warning('Could not determine subnet of Machine %s',
+                        machine['metadata']['name'])
+            return False
+
+        with lockutils.lock('kuryr-machine-add'):
+            if subnet_id not in self.subnets:
+                LOG.info('Adding subnet %s to the worker nodes subnets as '
+                         'machine %s runs in it.', subnet_id,
+                         machine['metadata']['name'])
+                self.subnets.add(subnet_id)
+                return True
+            return False
+
+    def delete_node(self, machine):
+        k8s = clients.get_kubernetes_client()
+        affected_subnet_id = self._get_subnet_from_machine(machine)
+        if not affected_subnet_id:
+            LOG.warning('Could not determine subnet of Machine %s',
+                        machine['metadata']['name'])
+            return False
+
+        machines = k8s.get(constants.OPENSHIFT_API_CRD_MACHINES)
+        for existing_machine in machines.get('items', []):
+            if affected_subnet_id == self._get_subnet_from_machine(
+                    existing_machine):
+                return False
+
+        # We know that subnet is no longer used, so we remove it.
+        LOG.info('Removing subnet %s from the worker nodes subnets',
+                 affected_subnet_id)
+        with lockutils.lock('kuryr-machine-add'):
+            self.subnets.remove(affected_subnet_id)
+
+        return True