Bug 1975676: Workaround OVN bug causing subports to be DOWN

kuryr_kubernetes/cni/handlers.py

@@ -36,7 +36,7 @@ def __init__(self, cni, on_done):
         self._callback = on_done
         self._vifs = {}
 
-    def on_present(self, pod):
+    def on_present(self, pod, *args, **kwargs):
         vifs = self._get_vifs(pod)
 
         if self.should_callback(pod, vifs):
@@ -107,7 +107,7 @@ def should_callback(self, kuryrport, vifs):
     def callback(self):
         self._callback(self._kuryrport, self._callback_vifs)
 
-    def on_deleted(self, kuryrport):
+    def on_deleted(self, kuryrport, *args, **kwargs):
         LOG.debug("Got kuryrport %s deletion event.",
                   kuryrport['metadata']['name'])
         if self._del_callback:

kuryr_kubernetes/controller/drivers/base.py

@@ -388,7 +388,7 @@ def release_vifs(self, pods, vifs, project_id=None, security_groups=None):
         raise NotImplementedError()
 
     @abc.abstractmethod
-    def activate_vif(self, vif):
+    def activate_vif(self, vif, **kwargs):
         """Updates VIF to become active.
 
         Implementing drivers should update the specified `vif` object's

kuryr_kubernetes/controller/drivers/nested_dpdk_vif.py

@@ -64,7 +64,7 @@ def release_vif(self, pod, vif, project_id=None, security_groups=None):
                         vif.id, vm_id)
             raise
 
-    def activate_vif(self, vif):
+    def activate_vif(self, vif, **kwargs):
         # NOTE(danil): new virtual interface was created in nova instance
         # during request_vif call, thus if it was not created successfully
         # an exception o_exc.SDKException would be throwed. During binding

kuryr_kubernetes/controller/drivers/nested_macvlan_vif.py

@@ -85,7 +85,7 @@ def release_vif(self, pod, vif, project_id=None, security_groups=None):
             LOG.warning("Unable to release port %s as it no longer exists.",
                         vif.id)
 
-    def activate_vif(self, vif):
+    def activate_vif(self, vif, **kwargs):
         # NOTE(mchiappe): there is no way to get feedback on the actual
         # interface creation or activation as no plugging can happen for this
         # interface type. However the status of the port is not relevant as

kuryr_kubernetes/controller/drivers/nested_vlan_vif.py

@@ -32,6 +32,7 @@
 
 DEFAULT_MAX_RETRY_COUNT = 3
 DEFAULT_RETRY_INTERVAL = 1
+ACTIVE_TIMEOUT = 90
 
 
 class NestedVlanPodVIFDriver(nested_vif.NestedPodVIFDriver):
@@ -113,6 +114,37 @@ def request_vifs(self, pod, project_id, subnets, security_groups,
             vifs.append(vif)
         return vifs
 
+    def activate_vif(self, vif, pod=None, retry_info=None):
+        try:
+            super().activate_vif(vif)
+        except k_exc.ResourceNotReady:
+            if retry_info and retry_info.get('elapsed', 0) > ACTIVE_TIMEOUT:
+                parent_port = self._get_parent_port(pod)
+                trunk_id = self._get_trunk_id(parent_port)
+                # NOTE(dulek): We don't need a lock to prevent VLAN ID from
+                #              being taken over because the SegmentationDriver
+                #              will keep it reserved in memory unless we
+                #              release it. And we won't.
+                LOG.warning('Subport %s is in DOWN status for more than %d '
+                            'seconds. This is a Neutron issue. Attempting to '
+                            'reattach the subport to trunk %s using VLAN ID %s'
+                            ' to fix it.', vif.id, retry_info['elapsed'],
+                            trunk_id, vif.vlan_id)
+                try:
+                    self._remove_subport(trunk_id, vif.id)
+                except os_exc.NotFoundException:
+                    # NOTE(dulek): This may happen when _add_subport() failed
+                    #              or Kuryr crashed between the calls. Let's
+                    #              try to fix it hoping that VLAN ID is still
+                    #              free.
+                    LOG.warning('Subport %s was not attached to the trunk. '
+                                'Trying to attach it anyway.', vif.id)
+                self._add_subport(trunk_id, vif.id,
+                                  requested_vlan_id=vif.vlan_id)
+                LOG.warning("Reattached subport %s, its state will be "
+                            "rechecked when event will be retried.", vif.id)
+            raise
+
     def release_vif(self, pod, vif, project_id=None, security_groups=None):
         os_net = clients.get_network_client()
         parent_port = self._get_parent_port(pod)
@@ -172,7 +204,7 @@ def _get_trunk_id(self, port):
                       "with a Neutron vlan trunk")
             raise k_exc.K8sNodeTrunkPortFailure
 
-    def _add_subport(self, trunk_id, subport):
+    def _add_subport(self, trunk_id, subport, requested_vlan_id=None):
         """Adds subport port to Neutron trunk
 
         This method gets vlanid allocated from kuryr segmentation driver.
@@ -186,29 +218,34 @@ def _add_subport(self, trunk_id, subport):
         os_net = clients.get_network_client()
         retry_count = 1
         while True:
-            try:
-                vlan_id = self._get_vlan_id(trunk_id)
-            except os_exc.SDKException:
-                LOG.error("Getting VlanID for subport on "
-                          "trunk %s failed!!", trunk_id)
-                raise
+            if requested_vlan_id:
+                vlan_id = requested_vlan_id
+            else:
+                try:
+                    vlan_id = self._get_vlan_id(trunk_id)
+                except os_exc.SDKException:
+                    LOG.error("Getting VLAN ID for subport on "
+                              "trunk %s failed!!", trunk_id)
+                    raise
+
             subport = [{'segmentation_id': vlan_id,
                         'port_id': subport,
                        'segmentation_type': 'vlan'}]
             try:
                 os_net.add_trunk_subports(trunk_id, subport)
             except os_exc.ConflictException:
-                if retry_count < DEFAULT_MAX_RETRY_COUNT:
-                    LOG.error("vlanid already in use on trunk, "
-                              "%s. Retrying...", trunk_id)
+                if (retry_count < DEFAULT_MAX_RETRY_COUNT and
+                        not requested_vlan_id):
+                    LOG.error("VLAN ID already in use on trunk %s. "
+                              "Retrying.", trunk_id)
                     retry_count += 1
                     sleep(DEFAULT_RETRY_INTERVAL)
                     continue
                 else:
-                    LOG.error(
-                        "MAX retry count reached. Failed to add subport")
+                    LOG.error("Failed to add subport %s to trunk %s due to "
+                              "VLAN ID %d conflict.", subport, trunk_id,
+                              vlan_id)
                     raise
-
             except os_exc.SDKException:
                 LOG.exception("Error happened during subport "
                               "addition to trunk %s", trunk_id)

kuryr_kubernetes/controller/drivers/neutron_vif.py

@@ -77,7 +77,7 @@ def request_vifs(self, pod, project_id, subnets, security_groups,
     def release_vif(self, pod, vif, project_id=None, security_groups=None):
         clients.get_network_client().delete_port(vif.id)
 
-    def activate_vif(self, vif):
+    def activate_vif(self, vif, **kwargs):
         if vif.active:
             return
 

kuryr_kubernetes/controller/drivers/sriov.py

@@ -69,7 +69,7 @@ def request_vif(self, pod, project_id, subnets, security_groups):
         self._reduce_remaining_sriov_vfs(pod, physnet)
         return vif
 
-    def activate_vif(self, vif):
+    def activate_vif(self, vif, **kwargs):
         vif.active = True
 
     def _get_physnet_subnet_mapping(self):

kuryr_kubernetes/controller/drivers/vif_pool.py

@@ -118,8 +118,8 @@ def request_vif(self, pod, project_id, subnets, security_groups):
     def release_vif(self, pod, vif, *argv):
         self._drv_vif.release_vif(pod, vif, *argv)
 
-    def activate_vif(self, vif):
-        self._drv_vif.activate_vif(vif)
+    def activate_vif(self, vif, **kwargs):
+        self._drv_vif.activate_vif(vif, **kwargs)
 
     def update_vif_sgs(self, pod, sgs):
         self._drv_vif.update_vif_sgs(pod, sgs)
@@ -168,8 +168,8 @@ def __init__(self):
     def set_vif_driver(self, driver):
         self._drv_vif = driver
 
-    def activate_vif(self, vif):
-        self._drv_vif.activate_vif(vif)
+    def activate_vif(self, vif, **kwargs):
+        self._drv_vif.activate_vif(vif, **kwargs)
 
     def update_vif_sgs(self, pod, sgs):
         self._drv_vif.update_vif_sgs(pod, sgs)
@@ -1230,9 +1230,9 @@ def release_vif(self, pod, vif, *argv):
         vif_drv_alias = self._get_vif_drv_alias(vif)
         self._vif_drvs[vif_drv_alias].release_vif(pod, vif, *argv)
 
-    def activate_vif(self, vif):
+    def activate_vif(self, vif, **kwargs):
         vif_drv_alias = self._get_vif_drv_alias(vif)
-        self._vif_drvs[vif_drv_alias].activate_vif(vif)
+        self._vif_drvs[vif_drv_alias].activate_vif(vif, **kwargs)
 
     def update_vif_sgs(self, pod, sgs):
         pod_vif_type = self._get_pod_vif_type(pod)

kuryr_kubernetes/controller/handlers/kuryrnetwork.py

@@ -49,7 +49,7 @@ def __init__(self):
             self._drv_svc_sg = (
                 drivers.ServiceSecurityGroupsDriver.get_instance())
 
-    def on_present(self, kuryrnet_crd):
+    def on_present(self, kuryrnet_crd, *args, **kwargs):
         ns_name = kuryrnet_crd['spec']['nsName']
         project_id = kuryrnet_crd['spec']['projectId']
         kns_status = kuryrnet_crd.get('status', {})
@@ -89,7 +89,7 @@ def on_present(self, kuryrnet_crd):
             status = {'nsLabels': kuryrnet_crd['spec']['nsLabels']}
             self._patch_kuryrnetwork_crd(kuryrnet_crd, status, labels=True)
 
-    def on_finalize(self, kuryrnet_crd):
+    def on_finalize(self, kuryrnet_crd, *args, **kwargs):
         LOG.debug("Deleting kuryrnetwork CRD resources: %s", kuryrnet_crd)
 
         net_id = kuryrnet_crd.get('status', {}).get('netId')

kuryr_kubernetes/controller/handlers/kuryrnetwork_population.py

@@ -42,7 +42,7 @@ def __init__(self):
             specific_driver='multi_pool')
         self._drv_vif_pool.set_vif_driver()
 
-    def on_present(self, kuryrnet_crd):
+    def on_present(self, kuryrnet_crd, *args, **kwargs):
         subnet_id = kuryrnet_crd.get('status', {}).get('subnetId')
         if not subnet_id:
             LOG.debug("No Subnet present for KuryrNetwork %s",

kuryr_kubernetes/controller/handlers/kuryrnetworkpolicy.py

@@ -111,7 +111,7 @@ def _find_sgs(self, a, rules):
 
         return False
 
-    def on_present(self, knp):
+    def on_present(self, knp, *args, **kwargs):
         uniq_name = utils.get_res_unique_name(knp)
         LOG.debug('on_present() for NP %s', uniq_name)
         project_id = self._drv_project.get_project(knp)
@@ -258,7 +258,7 @@ def _get_policy_net_id(self, knp):
             raise
         return net_crd['status']['netId']
 
-    def on_finalize(self, knp):
+    def on_finalize(self, knp, *args, **kwargs):
         LOG.debug("Finalizing KuryrNetworkPolicy %s")
         project_id = self._drv_project.get_project(knp)
         pods_to_update = self._drv_policy.affected_pods(knp)

kuryr_kubernetes/controller/handlers/kuryrport.py

@@ -60,14 +60,16 @@ def __init__(self):
                                 .get_instance())
         self.k8s = clients.get_kubernetes_client()
 
-    def on_present(self, kuryrport_crd):
+    def on_present(self, kuryrport_crd, *args, **kwargs):
         if not kuryrport_crd['status']['vifs']:
             # Get vifs
             if not self.get_vifs(kuryrport_crd):
                 # Ignore this event, according to one of the cases logged in
                 # get_vifs method.
                 return
 
+        retry_info = kwargs.get('retry_info')
+
         vifs = {ifname: {'default': data['default'],
                          'vif': objects.base.VersionedObject
                          .obj_from_primitive(data['vif'])}
@@ -77,6 +79,7 @@ def on_present(self, kuryrport_crd):
             return
 
         changed = False
+        pod = self._get_pod(kuryrport_crd)
 
         try:
             for ifname, data in vifs.items():
@@ -88,22 +91,14 @@ def on_present(self, kuryrport_crd):
                     driver_utils.update_port_pci_info(pod_node, data['vif'])
                 if not data['vif'].active:
                     try:
-                        self._drv_vif_pool.activate_vif(data['vif'])
+                        self._drv_vif_pool.activate_vif(data['vif'], pod=pod,
+                                                        retry_info=retry_info)
                         changed = True
                     except os_exc.ResourceNotFound:
                         LOG.debug("Port not found, possibly already deleted. "
                                   "No need to activate it")
         finally:
             if changed:
-                try:
-                    name = kuryrport_crd['metadata']['name']
-                    namespace = kuryrport_crd['metadata']['namespace']
-                    pod = self.k8s.get(f"{constants.K8S_API_NAMESPACES}"
-                                       f"/{namespace}/pods/{name}")
-                except k_exc.K8sResourceNotFound as ex:
-                    LOG.exception("Failed to get pod: %s", ex)
-                    raise
-
                 project_id = self._drv_project.get_project(pod)
 
                 try:
@@ -121,7 +116,8 @@ def on_present(self, kuryrport_crd):
                 try:
                     self._record_pod_creation_metric(pod)
                 except Exception:
-                    LOG.debug("Failed to record metric for pod %s", name)
+                    LOG.debug("Failed to record metric for pod %s",
+                              pod['metadata']['name'])
 
                 if driver_utils.is_network_policy_enabled():
                     crd_pod_selectors = self._drv_sg.create_sg_rules(pod)
@@ -130,7 +126,7 @@ def on_present(self, kuryrport_crd):
                         self._update_services(services, crd_pod_selectors,
                                               project_id)
 
-    def on_finalize(self, kuryrport_crd):
+    def on_finalize(self, kuryrport_crd, *args, **kwargs):
         name = kuryrport_crd['metadata']['name']
         namespace = kuryrport_crd['metadata']['namespace']
         try:
@@ -303,3 +299,13 @@ def _record_pod_creation_metric(self, pod):
                 pod_creation_time = datetime.datetime.now() - time_obj
                 pod_creation_sec = (pod_creation_time).total_seconds()
                 exporter.record_pod_creation_metric(pod_creation_sec)
+
+    def _get_pod(self, kuryrport_crd):
+        try:
+            name = kuryrport_crd['metadata']['name']
+            namespace = kuryrport_crd['metadata']['namespace']
+            return self.k8s.get(f"{constants.K8S_API_NAMESPACES}"
+                                f"/{namespace}/pods/{name}")
+        except k_exc.K8sResourceNotFound as ex:
+            LOG.exception("Failed to get pod: %s", ex)
+            raise

kuryr_kubernetes/controller/handlers/lbaas.py

@@ -52,7 +52,7 @@ def _bump_network_policies(self, svc):
         if driver_utils.is_network_policy_enabled():
             driver_utils.bump_networkpolicies(svc['metadata']['namespace'])
 
-    def on_present(self, service):
+    def on_present(self, service, *args, **kwargs):
         reason = self._should_ignore(service)
         if reason:
             LOG.debug(reason, service['metadata']['name'])
@@ -111,7 +111,7 @@ def _patch_service_finalizer(self, service):
         k8s = clients.get_kubernetes_client()
         return k8s.add_finalizer(service, k_const.SERVICE_FINALIZER)
 
-    def on_finalize(self, service):
+    def on_finalize(self, service, *args, **kwargs):
         k8s = clients.get_kubernetes_client()
 
         svc_name = service['metadata']['name']
@@ -271,7 +271,7 @@ def __init__(self):
                 self._lb_provider = (
                     config.CONF.kubernetes.endpoints_driver_octavia_provider)
 
-    def on_present(self, endpoints):
+    def on_present(self, endpoints, *args, **kwargs):
         if self._move_annotations_to_crd(endpoints):
             return
 

kuryr_kubernetes/controller/handlers/loadbalancer.py

@@ -54,7 +54,7 @@ def __init__(self):
         # We need to set the requested load balancer provider
         # according to 'endpoints_driver_octavia_provider' configuration.
 
-    def on_present(self, loadbalancer_crd):
+    def on_present(self, loadbalancer_crd, *args, **kwargs):
         if loadbalancer_crd.get('status', None) is None:
 
             kubernetes = clients.get_kubernetes_client()
@@ -124,7 +124,7 @@ def _has_pods(self, loadbalancer_crd):
                    for endpoint in ep_slice.get('endpoints', [])
                    if endpoint['targetRef'].get('kind', []) == 'Pod')
 
-    def on_finalize(self, loadbalancer_crd):
+    def on_finalize(self, loadbalancer_crd, *args, **kwargs):
         LOG.debug("Deleting the loadbalancer CRD")
 
         if loadbalancer_crd['status'] != {}:

kuryr_kubernetes/controller/handlers/namespace.py

@@ -71,7 +71,7 @@ def _upgrade_crds(self):
             except exceptions.K8sResourceNotFound:
                 LOG.debug('Kuryrnet object already deleted: %s', net_crd)
 
-    def on_present(self, namespace):
+    def on_present(self, namespace, *args, **kwargs):
         ns_labels = namespace['metadata'].get('labels', {})
         ns_name = namespace['metadata']['name']
         kns_crd = self._get_kns_crd(ns_name)

kuryr_kubernetes/controller/handlers/pod_label.py

@@ -46,7 +46,7 @@ def __init__(self):
         self._drv_vif_pool.set_vif_driver()
         self._drv_lbaas = drivers.LBaaSDriver.get_instance()
 
-    def on_present(self, pod):
+    def on_present(self, pod, *args, **kwargs):
         if driver_utils.is_host_network(pod) or not self._has_vifs(pod):
             # NOTE(ltomasbo): The event will be retried once the vif handler
             # annotates the pod with the pod state.