diff --git a/reproman/interface/jobs.py b/reproman/interface/jobs.py index cc4fb7119..30c317df4 100644 --- a/reproman/interface/jobs.py +++ b/reproman/interface/jobs.py @@ -14,12 +14,14 @@ import logging import yaml +from reproman.dochelpers import exc_str from reproman.interface.base import Interface from reproman.support.jobs.local_registry import LocalRegistry from reproman.support.jobs.orchestrators import ORCHESTRATORS from reproman.resource import get_manager from reproman.support.param import Parameter from reproman.support.constraints import EnsureChoice +from reproman.support.exceptions import OrchestratorError from reproman.support.exceptions import ResourceNotFoundError from reproman.utils import chpwd @@ -212,6 +214,8 @@ def __call__(queries, action="auto", all_=False, status=False): for job in jobs: try: fn(job) + except OrchestratorError as exc: + lgr.error("job %s failed: %s", job, exc_str(exc)) except ResourceNotFoundError as exc: lgr.error("Resource %s (%s) no longer exists", job["resource_id"], job["resource_name"]) diff --git a/reproman/interface/tests/test_run.py b/reproman/interface/tests/test_run.py index 37dfb9a57..3f8f21e5f 100644 --- a/reproman/interface/tests/test_run.py +++ b/reproman/interface/tests/test_run.py @@ -22,6 +22,7 @@ from reproman.utils import chpwd from reproman.utils import swallow_logs from reproman.utils import swallow_outputs +from reproman.support.exceptions import OrchestratorError from reproman.support.exceptions import ResourceNotFoundError from reproman.tests import fixtures from reproman.tests.utils import create_tree @@ -321,3 +322,25 @@ def test_jobs_deleted_resource(context): assert "todelete" not in output.out # ... but the alive one will. assert "myshell" in output.out + + +def test_jobs_orc_error(context): + run = context["run_fn"] + jobs = context["jobs_fn"] + registry = context["registry"] + + run(command=["doesntmatter1"], resref="myshell") + + jobfiles = registry.find_job_files() + assert len(jobfiles) == 1 + + with swallow_outputs() as output: + with swallow_logs(new_level=logging.ERROR) as log: + def die_orc(*args, **kwargs): + raise OrchestratorError("resurrection failed") + + with patch("reproman.interface.jobs.show_oneline", + side_effect=die_orc): + jobs(queries=[], status=True) + assert "myshell" not in output.out + assert "resurrection failed" in log.out