Add rollbar error logging to the celery worker

matl_online/app.py

@@ -1,14 +1,13 @@
 """The app module, containing the app factory function."""
 from typing import Optional, Type
 
-import rollbar  # type: ignore[import]
 from flask import Flask, got_request_exception
 from rollbar.contrib.flask import report_exception  # type: ignore[import]
 
 from matl_online import public
 from matl_online.assets import assets
 from matl_online.commands import register_commands
-from matl_online.extensions import celery, csrf, db, migrate, socketio
+from matl_online.extensions import celery, csrf, db, migrate, rollbar, socketio
 from matl_online.settings import Config, get_celery_configuration, get_config
 
 

matl_online/extensions.py

@@ -1,13 +1,16 @@
 """Enable third party extensions."""
 
+import rollbar  # type: ignore[import]
 from celery import Celery
-from flask_migrate import Migrate  # type: ignore
-from flask_socketio import SocketIO  # type: ignore
+from flask_migrate import Migrate  # type: ignore[import]
+from flask_socketio import SocketIO  # type: ignore[import]
 from flask_sqlalchemy import SQLAlchemy
-from flask_wtf import CSRFProtect  # type: ignore
+from flask_wtf import CSRFProtect  # type: ignore[import]
 
 db = SQLAlchemy()
 migrate = Migrate()
 socketio = SocketIO()
 celery = Celery(__name__)
 csrf = CSRFProtect()
+
+__all__ = ["rollbar", "db", "migrate", "socketio", "celery", "csrf"]

matl_online/tasks.py

@@ -10,10 +10,10 @@
 
 from celery import Task
 from celery.exceptions import SoftTimeLimitExceeded
-from celery.signals import worker_process_init
+from celery.signals import task_failure, worker_process_init
 from flask_socketio import SocketIO  # type: ignore
 
-from matl_online.extensions import celery
+from matl_online.extensions import celery, rollbar
 from matl_online.matl.core import matl
 from matl_online.matl.io import parse_matl_results
 from matl_online.octave import OctaveSession
@@ -186,6 +186,10 @@ def matl_task(
             task.handler.process_message("[STDERR]Operation timed out")
             task.on_term()
             raise
+        except Exception:
+            task.handler.process_message("[STDERR]Unknown error")
+            task.on_term()
+            raise
 
     return result
 
@@ -209,5 +213,10 @@ def _initialize_process(**kwargs: Any) -> None:
     )
 
 
+@task_failure.connect
+def handle_task_failure(**kwargs: Any) -> None:
+    rollbar.report_exc_info(extra_data=kwargs)
+
+
 # When a worker process is spawned, initialize octave
 worker_process_init.connect(_initialize_process)

tests/test_tasks.py

@@ -81,6 +81,44 @@ def test_normal(
         received = socketio_client.get_received()
         assert received[-1]["args"][0] == {"message": "", "success": True}
 
+    def test_exception(
+        self,
+        mocker: MockerFixture,
+        octave_mock: Mock,
+        socketio_client: SocketIOTestClient,
+    ) -> None:
+        """Ensure proper handling of keyboard interrupt events."""
+        socketio_client.get_received()
+
+        mocker.patch(
+            "matl_online.tasks.socket",
+            new_callable=_get_socketio_for_client(socketio_client),
+        )
+
+        ev = mocker.patch("matl_online.tasks.matl_task.octave.run")
+        ev.side_effect = Exception("Test")
+
+        matl_task.apply(
+            args=(
+                MATLRunTaskParameters(
+                    code="1D",
+                    version="20.0.0",
+                    session_id=session_id_for_client(socketio_client),
+                ),
+            ),
+        )
+
+        received = socketio_client.get_received()
+
+        payload = received[0]["args"][0]
+
+        assert payload.get("session") == session_id_for_client(socketio_client)
+        assert payload["data"][0]["type"] == "stderr"
+        assert payload["data"][0]["value"] == "Unknown error"
+
+        # Ultimately we alert the user that it failed
+        assert received[-1]["args"][0] == {"success": False}
+
     def test_keyboard_interrupt(
         self,
         mocker: MockerFixture,