Merge pull request #82 from idpass/fix/specify-ssl-context-for-xmlrpc

Allow optional TLS context for PDSSPP xmlrpc client

card_generator/cards/client.py

@@ -1,4 +1,5 @@
 import logging
+import ssl
 import xmlrpc.client
 from typing import Any, Literal, Optional
 
@@ -19,7 +20,7 @@ class OpenSPPClient:
     def __init__(self, server_root: str, username: str, password: str, db_name: str):
         """Initialize a OpenSPP client.
 
-        :param server_root: OpenSPP root url. E.g: https://dev.newlogic-demo.com/
+        :param server_root: OpenSPP root url. E.g: https://sample-server.com/
         :param username: User's username with access to the server
         :param password: User's password or API key with access to the server
         :param db_name: Name of database that contains the data for query
@@ -33,7 +34,11 @@ def __init__(self, server_root: str, username: str, password: str, db_name: str)
 
     @staticmethod
     def get_server_proxy(url):
-        return xmlrpc.client.ServerProxy(url)
+        context = None
+        if settings.OPENSPP_CUSTOM_TLS:
+            context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
+            context.load_verify_locations(settings.OPENSPP_CUSTOM_CERT_PATH)
+        return xmlrpc.client.ServerProxy(url, context=context)
 
     def login(self, username, password, kwargs: Optional[dict] = None):
         if not kwargs:
@@ -327,16 +332,10 @@ def get_id_queue_pdfs(self, batch_record: dict):
         )
 
     def update_queue_batch_record(self, batch_id: int, data: dict):
-        try:
-            response = self.call_api(
-                method_name="update",
-                model_name=settings.OPENSPP_QUEUE_BATCH_MODEL,
-                query_params=[[["id", "=", batch_id]]],
-                item_ids=[batch_id],
-                data=data,
-            )
-        except xmlrpc.client.Fault as e:
-            logger.info(f"Error in updating batch ID {batch_id}")
-            logger.info(e.faultString)
-            return
-        return response
+        return self.call_api(
+            method_name="update",
+            model_name=settings.OPENSPP_QUEUE_BATCH_MODEL,
+            query_params=[[["id", "=", batch_id]]],
+            item_ids=[batch_id],
+            data=data,
+        )

card_generator/tasks/cards.py

@@ -1,7 +1,8 @@
 import logging
 import tempfile
+import xmlrpc.client
 
-from celery import shared_task
+from celery import Task, shared_task
 from django.conf import settings
 from django.utils.timezone import now
 from PyPDF2 import PdfMerger
@@ -12,6 +13,38 @@
 logger = logging.getLogger(__name__)
 
 
+class OPENSPPCeleryTask(Task):
+    max_retries = settings.CELERY_MAX_RETRIES
+
+    def on_failure(self, exc, task_id, args, kwargs, einfo):
+        batch_id = args[1]["batch_id"]
+        data = {"merge_status": "error_merging"}
+        try:
+            client = QueueCardsClient(
+                server_root=settings.OPENSPP_SERVER_ROOT,
+                username=settings.OPENSPP_USERNAME,
+                password=settings.OPENSPP_API_TOKEN,
+                db_name=settings.OPENSPP_DB_NAME,
+            )
+        except Exception as e:  # noqa Lets catch all errors error for debugging
+            logger.info(
+                f"Error raised on client while updating batch {batch_id} status to failed. {str(e)}"
+            )
+            return
+        try:
+            client.update_queue_batch_record(batch_id=batch_id, data=data)
+        except (
+            xmlrpc.client.ProtocolError,
+            xmlrpc.client.Fault,
+            Exception,
+        ) as e:  # noqa Lets catch all errors error for debugging
+            logger.info(
+                f"Error raised while updating batch {batch_id} failed status. {str(e)}"
+            )
+            return
+        logger.info(f"Batch #{batch_id} have been updated with failed status.")
+
+
 def get_pdfs(client: QueueCardsClient, batch_record: dict) -> list:
     """
     Get the cards from OpenSPP
@@ -89,7 +122,7 @@ def perform_merging(client: QueueCardsClient, batch_id: int) -> None:
         )
 
 
-@shared_task(bind=True, max_retries=3)
+@shared_task(bind=True, base=OPENSPPCeleryTask)
 def merge_cards(self, batch_id: int) -> None:
     """
     Merge cards of a Batch Queue from OpenSPP server.
@@ -109,14 +142,14 @@ def merge_cards(self, batch_id: int) -> None:
         )
     except Exception as e:  # noqa Lets catch all errors error for debugging and retry
         logger.info(f"Error raised on client. {str(e)}")
-        self.retry(exc=e, countdown=30)
-        return
+        raise self.retry(exc=e, countdown=settings.CELERY_RETRY_COUNTDOWN)
     try:
         perform_merging(client, batch_id)
-    except Exception as e:  # noqa Lets catch all errors error for debugging and retry
+    except (
+        xmlrpc.client.ProtocolError,
+        xmlrpc.client.Fault,
+        Exception,
+    ) as e:  # noqa Lets catch all errors error for debugging and retry
         logger.info(f"Error raised while performing merge. {str(e)}")
-        data = {"merge_status": "error_merging"}
-        client.update_queue_batch_record(batch_id=batch_id, data=data)
-        self.retry(exc=e, countdown=30)
-        return
+        raise self.retry(countdown=settings.CELERY_RETRY_COUNTDOWN)
     logger.info(f"Batch #{batch_id} have been updated with merged cards.")

config/settings/base.py

@@ -318,3 +318,12 @@
 OPENSPP_DEFAULT_FETCH_LIMIT = env.str("OPENSPP_DEFAULT_FETCH_LIMIT", default=10)
 OPENSPP_DEFAULT_CARD_X_DPI = env.str("OPENSPP_DEFAULT_CARD_X_DPI", default="72")
 OPENSPP_DEFAULT_CARD_Y_DPI = env.str("OPENSPP_DEFAULT_CARD_Y_DPI", default="72")
+
+# TLS
+OPENSPP_CUSTOM_TLS = env.bool("OPENSPP_USE_TLS", default=False)
+if OPENSPP_CUSTOM_TLS:
+    OPENSPP_CUSTOM_CERT_PATH = env.str("OPENSPP_CUSTOM_CERT_PATH")
+
+# Celery
+CELERY_MAX_RETRIES = env.int("CELERY_MAX_RETRIES", default=3)
+CELERY_RETRY_COUNTDOWN = env.int("CELERY_RETRY_COUNTDOWN", default=30)

production.yml

@@ -18,6 +18,8 @@ services:
       - "8000:8000"
     volumes:
       - open_card_generator_prod_media:/app/card_generator/media
+    extra_hosts:
+      - "${OPENSPP_HOSTNAME}:${OPENSPP_IP_ADDRESS}"
 
   celery:
     <<: *django
@@ -28,6 +30,7 @@ services:
     command: celery -A config.celery_app worker -l info -E
     ports: []
     volumes: []
+    restart: always
 
   redis:
     image: redis