From 0ccd9445238c50a37b420d4d7ea5a9fdd0db03f4 Mon Sep 17 00:00:00 2001
From: Lindy Lindstrom <llindstrom@lco.global>
Date: Wed, 1 Mar 2023 20:06:01 +0000
Subject: [PATCH] catch all the Exceptions; log errors and warnings accordingly

Lots can go wrong in
helpix_utils.create_localization_for_multiorder_fits
b/c it downloads and processes a fits file from GraceDB.
Uncaught exceptions were putting the Kafka client in a bad
state. So, we catch everything here to avoid that.
---
 .../alertstream_handlers/gw_event_handler.py  | 20 ++++++++++++++-----
 1 file changed, 15 insertions(+), 5 deletions(-)

diff --git a/tom_nonlocalizedevents/alertstream_handlers/gw_event_handler.py b/tom_nonlocalizedevents/alertstream_handlers/gw_event_handler.py
index 25e0d1b..694e1f8 100644
--- a/tom_nonlocalizedevents/alertstream_handlers/gw_event_handler.py
+++ b/tom_nonlocalizedevents/alertstream_handlers/gw_event_handler.py
@@ -61,12 +61,18 @@ def handle_message(message):
         if nle_created:
             logger.info(f"Ingested a new GW event with id {fields['TRIGGER_NUM']} from alertstream")
         # Next attempt to ingest and build the localization of the event
-        localization = create_localization_for_multiorder_fits(
-            nonlocalizedevent=nonlocalizedevent,
-            multiorder_fits_url=get_moc_url_from_skymap_fits_url(fields['SKYMAP_FITS_URL'])
-        )
+        try:
+            localization = create_localization_for_multiorder_fits(
+                nonlocalizedevent=nonlocalizedevent,
+                multiorder_fits_url=get_moc_url_from_skymap_fits_url(fields['SKYMAP_FITS_URL'])
+            )
+        except Exception as e:
+            localization = None
+            logger.error(f'Could not create EventLocalization for messsage: {fields}. Exception: {e}')
+            logger.error(traceback.format_exc())
+
         # Now ingest the sequence for that event
-        EventSequence.objects.update_or_create(
+        event_sequence, es_created = EventSequence.objects.update_or_create(
             nonlocalizedevent=nonlocalizedevent,
             localization=localization,
             sequence_id=fields['SEQUENCE_NUM'],
@@ -74,6 +80,10 @@ def handle_message(message):
                 'event_subtype': fields['NOTICE_TYPE']
             }
         )
+        if es_created and localization is None:
+            warning_msg = (f'{"Creating" if es_created else "Updating"} EventSequence without EventLocalization:'
+                           f'{event_sequence} for NonLocalizedEvent: {nonlocalizedevent}')
+            logger.warning(warning_msg)
 
 
 def handle_retraction(message):