Skip to content
Permalink
Browse files

BF: jobs: Treat deleted local directory as orchestrator error

Trying to resurrect the orchestrator for a job where the local working
directory has been deleted leads to a FileNotFoundError.  Recast this
exception as an OrchestratorError so that (1) we can provide a more
helpful message and (2) the jobs() call can continue with any other
jobs specified rather than aborting.
  • Loading branch information
kyleam committed Jan 2, 2020
1 parent b16279a commit c7fdb3b03e284c452b97602f9eaa38eda182c737
Showing with 23 additions and 1 deletion.
  1. +10 −1 reproman/interface/jobs.py
  2. +13 −0 reproman/interface/tests/test_run.py
@@ -72,7 +72,16 @@ def match(query_id, jobids):

def _resurrect_orc(job):
resource = get_manager().get_resource(job["resource_id"], "id")
with chpwd(job["local_directory"]):
try:
# Create chpwd separately so that this try-except block doesn't cover
# the context manager suite below.
cd = chpwd(job["local_directory"])
except FileNotFoundError:
raise OrchestratorError(
"local directory for job {} no longer exists: {}"
.format(job["_jobid"], job["local_directory"]))

with cd:
orchestrator_class = ORCHESTRATORS[job["orchestrator"]]
orc = orchestrator_class(resource, job["submitter"], job,
resurrection=True)
@@ -13,6 +13,7 @@
from unittest.mock import patch
import os
import os.path as op
import shutil
import time

import pytest
@@ -455,6 +456,18 @@ def test_jobs_deleted_resource(context):
assert "myshell" in output.out


def test_jobs_deleted_local_directory(context):
path = context["directory"]
run = context["run_fn"]
jobs = context["jobs_fn"]

run(command=["touch", "ok"], outputs=["ok"], resref="myshell")
shutil.rmtree(path)
with swallow_logs(new_level=logging.ERROR) as log:
jobs(queries=[], status=True)
assert "no longer exists" in log.out


def test_jobs_orc_error(context):
run = context["run_fn"]
jobs = context["jobs_fn"]

0 comments on commit c7fdb3b

Please sign in to comment.
You can’t perform that action at this time.