fix: Refactor for card merge task; fix issue with retry

Signed-off-by: Rommel Terrence Juanillo <terrence@newlogic.com>

card_generator/cards/client.py

@@ -332,16 +332,10 @@ def get_id_queue_pdfs(self, batch_record: dict):
         )
 
     def update_queue_batch_record(self, batch_id: int, data: dict):
-        try:
-            response = self.call_api(
-                method_name="update",
-                model_name=settings.OPENSPP_QUEUE_BATCH_MODEL,
-                query_params=[[["id", "=", batch_id]]],
-                item_ids=[batch_id],
-                data=data,
-            )
-        except xmlrpc.client.Fault as e:
-            logger.info(f"Error in updating batch ID {batch_id}")
-            logger.info(e.faultString)
-            return
-        return response
+        return self.call_api(
+            method_name="update",
+            model_name=settings.OPENSPP_QUEUE_BATCH_MODEL,
+            query_params=[[["id", "=", batch_id]]],
+            item_ids=[batch_id],
+            data=data,
+        )

card_generator/tasks/cards.py

@@ -1,7 +1,8 @@
 import logging
 import tempfile
+import xmlrpc.client
 
-from celery import shared_task
+from celery import Task, shared_task
 from django.conf import settings
 from django.utils.timezone import now
 from PyPDF2 import PdfMerger
@@ -12,6 +13,38 @@
 logger = logging.getLogger(__name__)
 
 
+class OPENSPPCeleryTask(Task):
+    max_retries = 3
+
+    def on_failure(self, exc, task_id, args, kwargs, einfo):
+        batch_id = args[1]["batch_id"]
+        data = {"merge_status": "error_merging"}
+        try:
+            client = QueueCardsClient(
+                server_root=settings.OPENSPP_SERVER_ROOT,
+                username=settings.OPENSPP_USERNAME,
+                password=settings.OPENSPP_API_TOKEN,
+                db_name=settings.OPENSPP_DB_NAME,
+            )
+        except Exception as e:  # noqa Lets catch all errors error for debugging
+            logger.info(
+                f"Error raised on client while updating batch {batch_id} status to failed. {str(e)}"
+            )
+            return
+        try:
+            client.update_queue_batch_record(batch_id=batch_id, data=data)
+        except (
+            xmlrpc.client.ProtocolError,
+            xmlrpc.client.Fault,
+            Exception,
+        ) as e:  # noqa Lets catch all errors error for debugging
+            logger.info(
+                f"Error raised while updating batch {batch_id} failed status. {str(e)}"
+            )
+            return
+        logger.info(f"Batch #{batch_id} have been updated with failed status.")
+
+
 def get_pdfs(client: QueueCardsClient, batch_record: dict) -> list:
     """
     Get the cards from OpenSPP
@@ -89,7 +122,7 @@ def perform_merging(client: QueueCardsClient, batch_id: int) -> None:
         )
 
 
-@shared_task(bind=True, max_retries=3)
+@shared_task(bind=True, base=OPENSPPCeleryTask)
 def merge_cards(self, batch_id: int) -> None:
     """
     Merge cards of a Batch Queue from OpenSPP server.
@@ -109,14 +142,14 @@ def merge_cards(self, batch_id: int) -> None:
         )
     except Exception as e:  # noqa Lets catch all errors error for debugging and retry
         logger.info(f"Error raised on client. {str(e)}")
-        self.retry(exc=e, countdown=30)
-        return
+        raise self.retry(exc=e, countdown=30)
     try:
         perform_merging(client, batch_id)
-    except Exception as e:  # noqa Lets catch all errors error for debugging and retry
+    except (
+        xmlrpc.client.ProtocolError,
+        xmlrpc.client.Fault,
+        Exception,
+    ) as e:  # noqa Lets catch all errors error for debugging and retry
         logger.info(f"Error raised while performing merge. {str(e)}")
-        data = {"merge_status": "error_merging"}
-        client.update_queue_batch_record(batch_id=batch_id, data=data)
-        self.retry(exc=e, countdown=30)
-        return
+        raise self.retry(countdown=30)
     logger.info(f"Batch #{batch_id} have been updated with merged cards.")

production.yml

@@ -30,6 +30,7 @@ services:
     command: celery -A config.celery_app worker -l info -E
     ports: []
     volumes: []
+    restart: always
 
   redis:
     image: redis