add --pdb option to drop into debugger on exception

When we get an exception, sometimes it's not enough to just
see the backtrace, but we also want to get a debugger prompt
at the point of the exception.

Replaced the --do-raise option (apparently unused, and often
associated with getting a debugger with Gen2) with --pdb
(well-known and appreciated option from pytest), which causes
MPGraphExecutor._executeQuantaInProcess to drop into pdb.

doc/changes/DM-34215.feature.rst

@@ -0,0 +1 @@
+Replaced the unused `--do-raise` option with `--pdb`, which drops the user into the debugger on an exception.

python/lsst/ctrl/mpexec/cli/opt/optionGroups.py

@@ -118,7 +118,7 @@ def __init__(self):
         self.decorators = [
             option_section(sectionText="Execution options:"),
             ctrlMpExecOpts.clobber_outputs_option(),
-            ctrlMpExecOpts.do_raise_option(),
+            ctrlMpExecOpts.pdb_option(),
             ctrlMpExecOpts.profile_option(),
             dafButlerOpts.processes_option(),
             ctrlMpExecOpts.start_method_option(),

python/lsst/ctrl/mpexec/cli/opt/options.py

@@ -76,8 +76,8 @@ def _to_int(value):
 )
 
 
-do_raise_option = MWOptionDecorator(
-    "--do-raise", help="Raise an exception on error. (else log a message and continue?)", is_flag=True
+pdb_option = MWOptionDecorator(
+    "--pdb", help="Drop into pdb on exception (only if single process)?", is_flag=True
 )
 
 

python/lsst/ctrl/mpexec/cli/script/run.py

@@ -28,7 +28,7 @@
 
 
 def run(
-    do_raise,
+    pdb,
     graph_fixup,
     init_only,
     no_versions,
@@ -63,8 +63,8 @@ def run(
 
     Parameters
     ----------
-    do_raise : `bool`
-        Raise an exception in the case of an error.
+    pdb : `bool`
+        Drop into pdb on exception?
     graph_fixup : `str`
         The name of the class or factory method which makes an instance used
         for execution graph fixup.
@@ -154,7 +154,7 @@ def run(
         which ingore these unused kwargs.
     """
     args = SimpleNamespace(
-        do_raise=do_raise,
+        pdb=pdb,
         graph_fixup=graph_fixup,
         init_only=init_only,
         no_versions=no_versions,

python/lsst/ctrl/mpexec/cmdLineFwk.py

@@ -729,6 +729,7 @@ def runPipeline(self, graph, taskFactory, args, butler=None):
                 startMethod=args.start_method,
                 quantumExecutor=quantumExecutor,
                 failFast=args.fail_fast,
+                pdb=args.pdb,
                 executionGraphFixup=graphFixup,
             )
             try:

python/lsst/ctrl/mpexec/mpGraphExecutor.py

@@ -342,12 +342,14 @@ def __init__(
         *,
         startMethod=None,
         failFast=False,
+        pdb=False,
         executionGraphFixup=None,
     ):
         self.numProc = numProc
         self.timeout = timeout
         self.quantumExecutor = quantumExecutor
         self.failFast = failFast
+        self.pdb = pdb
         self.executionGraphFixup = executionGraphFixup
         self.report: Optional[Report] = None
 
@@ -436,6 +438,10 @@ def _executeQuantaInProcess(self, graph, butler):
                 self.quantumExecutor.execute(qnode.taskDef, qnode.quantum, butler)
                 successCount += 1
             except Exception as exc:
+                if self.pdb and self.numProc == 1:
+                    import pdb  # noqa E402: only load this when we need it
+
+                    pdb.post_mortem(exc.__traceback__)
                 failedNodes.add(qnode)
                 self.report.status = ExecutionStatus.FAILURE
                 if self.failFast: