RF: jobs: Continue if action raises OrchestratorError

We're bound to hit this at some point.  Log it and continue with the
next job.

reproman/interface/jobs.py

@@ -14,12 +14,14 @@
 import logging
 import yaml
 
+from reproman.dochelpers import exc_str
 from reproman.interface.base import Interface
 from reproman.support.jobs.local_registry import LocalRegistry
 from reproman.support.jobs.orchestrators import ORCHESTRATORS
 from reproman.resource import get_manager
 from reproman.support.param import Parameter
 from reproman.support.constraints import EnsureChoice
+from reproman.support.exceptions import OrchestratorError
 from reproman.support.exceptions import ResourceNotFoundError
 from reproman.utils import chpwd
 
@@ -212,6 +214,8 @@ def __call__(queries, action="auto", all_=False, status=False):
             for job in jobs:
                 try:
                     fn(job)
+                except OrchestratorError as exc:
+                    lgr.error("job %s failed: %s", job, exc_str(exc))
                 except ResourceNotFoundError as exc:
                     lgr.error("Resource %s (%s) no longer exists",
                               job["resource_id"], job["resource_name"])

reproman/interface/tests/test_run.py

@@ -22,6 +22,7 @@
 from reproman.utils import chpwd
 from reproman.utils import swallow_logs
 from reproman.utils import swallow_outputs
+from reproman.support.exceptions import OrchestratorError
 from reproman.support.exceptions import ResourceNotFoundError
 from reproman.tests import fixtures
 from reproman.tests.utils import create_tree
@@ -321,3 +322,25 @@ def test_jobs_deleted_resource(context):
             assert "todelete" not in output.out
             # ... but the alive one will.
             assert "myshell" in output.out
+
+
+def test_jobs_orc_error(context):
+    run = context["run_fn"]
+    jobs = context["jobs_fn"]
+    registry = context["registry"]
+
+    run(command=["doesntmatter1"], resref="myshell")
+
+    jobfiles = registry.find_job_files()
+    assert len(jobfiles) == 1
+
+    with swallow_outputs() as output:
+        with swallow_logs(new_level=logging.ERROR) as log:
+            def die_orc(*args, **kwargs):
+                raise OrchestratorError("resurrection failed")
+
+            with patch("reproman.interface.jobs.show_oneline",
+                       side_effect=die_orc):
+                jobs(queries=[], status=True)
+            assert "myshell" not in output.out
+            assert "resurrection failed" in log.out