Ignore 422 Unprocessable Entity on KLB patching

If we're in the process of handling a KuryrLoadBalancer event and
somebody's tries to delete the `.status` field there, any update we do
on `.status` will result in 422 Unprocessable Entity returned from K8s
API. There's no sense in failing kuryr-controller in that case as the
update should trigger another event and Kuryr should be able to recover
by discovering all the resources and filling the status again.

This commit fixes that by making sure in such situation we stop
processing the event and make the handler wait for the next one.

As a consuequence KLB's 404 returned from patch operations is also
handled as hard stop for further processing of the KuryrLoadBalancer as
there's no point in creating or deleting Octavia resources if the LB is
destined to be cascade deleted.

Change-Id: I5dabded04302268e2c5c25f6c31a5619cd0c28e1
Closes-Bug: 1921109

kuryr_kubernetes/controller/handlers/loadbalancer.py

@@ -60,7 +60,7 @@ def on_present(self, loadbalancer_crd):
             kubernetes = clients.get_kubernetes_client()
             try:
                 kubernetes.patch_crd('status',
-                                     utils.get_res_link(loadbalancer_crd),
+                                     loadbalancer_crd['metadata']['selfLink'],
                                      {})
             except k_exc.K8sResourceNotFound:
                 LOG.debug('KuryrLoadbalancer CRD not found %s',
@@ -109,18 +109,7 @@ def on_present(self, loadbalancer_crd):
                     loadbalancer_crd['status'][
                         'service_pub_ip_info'] = service_pub_ip_info
                     self._update_lb_status(loadbalancer_crd)
-                    kubernetes = clients.get_kubernetes_client()
-                    try:
-                        kubernetes.patch_crd('status', loadbalancer_crd[
-                            'metadata']['selfLink'], loadbalancer_crd[
-                                'status'])
-                    except k_exc.K8sResourceNotFound:
-                        LOG.debug('KuryrLoadbalancer CRD not found %s',
-                                  loadbalancer_crd)
-                    except k_exc.K8sClientException:
-                        LOG.exception('Error updating KuryLoadbalancer CRD %s',
-                                      loadbalancer_crd)
-                        raise
+                    self._patch_status(loadbalancer_crd)
 
     def _should_ignore(self, loadbalancer_crd):
         return not(self._has_pods(loadbalancer_crd) or
@@ -181,6 +170,25 @@ def on_finalize(self, loadbalancer_crd):
                           'for %s', service["metadata"]["name"])
             raise
 
+    def _patch_status(self, loadbalancer_crd):
+        kubernetes = clients.get_kubernetes_client()
+        try:
+            kubernetes.patch_crd(
+                'status', loadbalancer_crd['metadata']['selfLink'],
+                loadbalancer_crd['status'])
+        except k_exc.K8sResourceNotFound:
+            LOG.debug('KuryrLoadBalancer CRD not found %s', loadbalancer_crd)
+            return False
+        except k_exc.K8sUnprocessableEntity:
+            LOG.warning('KuryrLoadBalancer %s modified, retrying later.',
+                        utils.get_res_unique_name(loadbalancer_crd))
+            return False
+        except k_exc.K8sClientException:
+            LOG.exception('Error updating KuryLoadbalancer CRD %s',
+                          loadbalancer_crd)
+            raise
+        return True
+
     def _sync_lbaas_members(self, loadbalancer_crd):
         changed = False
 
@@ -228,8 +236,7 @@ def _sync_lbaas_sgs(self, klb_crd):
                       'due to missing loadbalancer field.')
             return None
         except k_exc.K8sClientException:
-            LOG.exception('Error syncing KuryrLoadBalancer'
-                          ' %s', svc_name)
+            LOG.exception('Error syncing KuryrLoadBalancer %s', svc_name)
             raise
         return klb_crd
 
@@ -338,18 +345,8 @@ def _add_new_members(self, loadbalancer_crd):
                         loadbalancer_crd['status']['members'] = []
                         loadbalancer_crd['status'].get('members', []).append(
                             member)
-                    kubernetes = clients.get_kubernetes_client()
-                    try:
-                        kubernetes.patch_crd('status', loadbalancer_crd[
-                            'metadata']['selfLink'], loadbalancer_crd[
-                                'status'])
-                    except k_exc.K8sResourceNotFound:
-                        LOG.debug('KuryrLoadbalancer CRD not found %s',
-                                  loadbalancer_crd)
-                    except k_exc.K8sClientException:
-                        LOG.exception('Error updating KuryLoadbalancer CRD %s',
-                                      loadbalancer_crd)
-                        raise
+                    if not self._patch_status(loadbalancer_crd):
+                        return False
                     changed = True
         return changed
 
@@ -425,18 +422,8 @@ def _remove_unused_members(self, loadbalancer_crd):
                        if m['id'] not in removed_ids]
             loadbalancer_crd['status']['members'] = members
 
-            kubernetes = clients.get_kubernetes_client()
-            try:
-                kubernetes.patch_crd('status', loadbalancer_crd[
-                    'metadata']['selfLink'], loadbalancer_crd[
-                        'status'])
-            except k_exc.K8sResourceNotFound:
-                LOG.debug('KuryrLoadbalancer CRD not found %s',
-                          loadbalancer_crd)
-            except k_exc.K8sClientException:
-                LOG.exception('Error updating KuryLoadbalancer CRD %s',
-                              loadbalancer_crd)
-                raise
+            if not self._patch_status(loadbalancer_crd):
+                return False
         return bool(removed_ids)
 
     def _sync_lbaas_pools(self, loadbalancer_crd):
@@ -474,17 +461,9 @@ def _add_new_pools(self, loadbalancer_crd):
                 loadbalancer_crd['status']['pools'] = []
                 loadbalancer_crd['status'].get('pools', []).append(
                     pool)
-            kubernetes = clients.get_kubernetes_client()
-            try:
-                kubernetes.patch_crd('status', loadbalancer_crd['metadata'][
-                    'selfLink'], loadbalancer_crd['status'])
-            except k_exc.K8sResourceNotFound:
-                LOG.debug('KuryrLoadbalancer CRD not found %s',
-                          loadbalancer_crd)
-            except k_exc.K8sClientException:
-                LOG.exception('Error updating KuryrLoadbalancer CRD %s',
-                              loadbalancer_crd)
-                raise
+
+            if not self._patch_status(loadbalancer_crd):
+                return False
             changed = True
         return changed
 
@@ -521,18 +500,8 @@ def _remove_unused_pools(self, loadbalancer_crd):
                                                      if m['pool_id'] not in
                                                      removed_ids]
 
-            kubernetes = clients.get_kubernetes_client()
-            try:
-                kubernetes.patch_crd('status', loadbalancer_crd[
-                    'metadata']['selfLink'], loadbalancer_crd[
-                        'status'])
-            except k_exc.K8sResourceNotFound:
-                LOG.debug('KuryrLoadbalancer CRD not found %s',
-                          loadbalancer_crd)
-            except k_exc.K8sClientException:
-                LOG.exception('Error updating KuryLoadbalancer CRD %s',
-                              loadbalancer_crd)
-                raise
+            if not self._patch_status(loadbalancer_crd):
+                return False
         return bool(removed_ids)
 
     def _sync_lbaas_listeners(self, loadbalancer_crd):
@@ -593,17 +562,8 @@ def _add_new_listeners(self, loadbalancer_crd):
                     loadbalancer_crd['status'].get('listeners', []).append(
                         listener)
 
-                kubernetes = clients.get_kubernetes_client()
-                try:
-                    kubernetes.patch_crd('status', loadbalancer_crd[
-                        'metadata']['selfLink'], loadbalancer_crd['status'])
-                except k_exc.K8sResourceNotFound:
-                    LOG.debug('KuryrLoadbalancer CRD not found %s',
-                              loadbalancer_crd)
-                except k_exc.K8sClientException:
-                    LOG.exception('Error updating KuryrLoadbalancer CRD %s',
-                                  loadbalancer_crd)
-                    raise
+                if not self._patch_status(loadbalancer_crd):
+                    return False
                 changed = True
         return changed
 
@@ -623,18 +583,8 @@ def _remove_unused_listeners(self, loadbalancer_crd):
                                                           []) if l['id']
                 not in removed_ids]
 
-            kubernetes = clients.get_kubernetes_client()
-            try:
-                kubernetes.patch_crd('status', loadbalancer_crd[
-                    'metadata']['selfLink'], loadbalancer_crd[
-                        'status'])
-            except k_exc.K8sResourceNotFound:
-                LOG.debug('KuryrLoadbalancer CRD not found %s',
-                          loadbalancer_crd)
-            except k_exc.K8sClientException:
-                LOG.exception('Error updating KuryLoadbalancer CRD %s',
-                              loadbalancer_crd)
-                raise
+            if not self._patch_status(loadbalancer_crd):
+                return False
         return bool(removed_ids)
 
     def _update_lb_status(self, lb_crd):
@@ -658,7 +608,6 @@ def _update_lb_status(self, lb_crd):
             raise
 
     def _sync_lbaas_loadbalancer(self, loadbalancer_crd):
-        changed = False
         lb = loadbalancer_crd['status'].get('loadbalancer')
 
         if lb and lb['ip'] != loadbalancer_crd['spec'].get('ip'):
@@ -697,20 +646,8 @@ def _sync_lbaas_loadbalancer(self, loadbalancer_crd):
                     provider=loadbalancer_crd['spec'].get('provider'))
                 loadbalancer_crd['status']['loadbalancer'] = lb
 
-            kubernetes = clients.get_kubernetes_client()
-            try:
-                kubernetes.patch_crd('status', loadbalancer_crd['metadata'][
-                    'selfLink'], loadbalancer_crd['status'])
-            except k_exc.K8sResourceNotFound:
-                LOG.debug('KuryrLoadbalancer CRD not found %s',
-                          loadbalancer_crd)
-            except k_exc.K8sClientException:
-                LOG.exception('Error updating KuryrLoadbalancer CRD %s',
-                              loadbalancer_crd)
-                raise
-            changed = True
-
-        return changed
+            return self._patch_status(loadbalancer_crd)
+        return False
 
     def _ensure_release_lbaas(self, loadbalancer_crd):
         attempts = 0
@@ -739,17 +676,7 @@ def _ensure_release_lbaas(self, loadbalancer_crd):
                 retry = True
 
             loadbalancer_crd['status'] = {}
-            k8s = clients.get_kubernetes_client()
-            try:
-                k8s.patch_crd('status', loadbalancer_crd['metadata'][
-                    'selfLink'], loadbalancer_crd['status'])
-            except k_exc.K8sResourceNotFound:
-                LOG.debug('KuryrLoadbalancer CRD not found %s',
-                          loadbalancer_crd)
-            except k_exc.K8sClientException:
-                LOG.exception('Error updating KuryrLoadbalancer CRD %s',
-                              loadbalancer_crd)
-                raise
+            self._patch_status(loadbalancer_crd)
             # NOTE(ltomasbo): give some extra time to ensure the Load
             # Balancer VIP is also released
             time.sleep(1)