Skip to content

Commit

Permalink
Merge pull request #404 from pepkit/reduce_tests
Browse files Browse the repository at this point in the history 
Reduce tests to use less subprocesses
  • Loading branch information
@donaldcampbelljr
donaldcampbelljr committed Aug 15, 2023
2 parents 8fbfb02 + 6e6923f commit dabcd65
Show file tree
Hide file tree
Showing 2 changed files with 82 additions and 51 deletions.
28 changes: 17 additions & 11 deletions looper/looper.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -415,6 +415,7 @@ def __call__(self, args, rerun=False, **compute_kwargs):
:param bool rerun: whether the given sample is being rerun rather than
run for the first time
"""
self.debug = {} # initialize empty dict for return values
max_cmds = sum(list(map(len, self.prj._samples_by_interface.values())))
self.counter.total = max_cmds
failures = defaultdict(list) # Collect problems by sample.
Expand Down Expand Up @@ -518,20 +519,25 @@ def __call__(self, args, rerun=False, **compute_kwargs):
)
)
_LOGGER.info("Commands submitted: {} of {}".format(cmd_sub_total, max_cmds))
self.debug["Commands submitted"] = "Commands submitted: {} of {}".format(
cmd_sub_total, max_cmds
)
if args.dry_run:
job_sub_total_if_real = job_sub_total
job_sub_total = 0
_LOGGER.info(
f"Dry run. No jobs were actually submitted, but {job_sub_total_if_real} would have been."
)
_LOGGER.info("Jobs submitted: {}".format(job_sub_total))
self.debug["Jobs submitted"] = job_sub_total

# Restructure sample/failure data for display.
samples_by_reason = defaultdict(set)
# Collect names of failed sample(s) by failure reason.
for sample, failures in failures.items():
for f in failures:
samples_by_reason[f].add(sample)
self.debug[f] = sample
# Collect samples by pipeline with submission failure.
for piface, conductor in submission_conductors.items():
# Don't add failure key if there are no samples that failed for
Expand Down Expand Up @@ -566,6 +572,8 @@ def __call__(self, args, rerun=False, **compute_kwargs):
_LOGGER.debug("Raising SampleFailedException")
raise SampleFailedException

return self.debug


class Reporter(Executor):
"""Combine project outputs into a browsable HTML report"""
Expand Down Expand Up @@ -1043,16 +1051,14 @@ def main(test_args=None):
sys.exit(1)

if args.command == "init":
sys.exit(
int(
not init_dotfile(
dotfile_path(),
args.config_file,
args.output_dir,
args.sample_pipeline_interfaces,
args.project_pipeline_interfaces,
args.force,
)
return int(
not init_dotfile(
dotfile_path(),
args.config_file,
args.output_dir,
args.sample_pipeline_interfaces,
args.project_pipeline_interfaces,
args.force,
)
)

Expand Down Expand Up @@ -1157,7 +1163,7 @@ def main(test_args=None):
run = Runner(prj)
try:
compute_kwargs = _proc_resources_spec(args)
run(args, rerun=(args.command == "rerun"), **compute_kwargs)
return run(args, rerun=(args.command == "rerun"), **compute_kwargs)
except SampleFailedException:
sys.exit(1)
except IOError:
Expand Down
105 changes: 65 additions & 40 deletions tests/smoketests/test_run.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -124,10 +124,13 @@ def test_looper_single_pipeline(self, prep_temp_pep):
PIPELINE_INTERFACES_KEY
] = pifaces[1]

stdout, stderr, rc = subp_exec(tp, "run")
print_standard_stream(stderr)
assert rc == 0
assert "Commands submitted: 6 of 6" not in str(stderr)
x = test_args_expansion(tp, "run")
try:
result = main(test_args=x)
print(result)
assert result["Commands submitted"] != "Commands submitted: 6 of 6"
except Exception:
raise pytest.fail("DID RAISE {0}".format(Exception))

def test_looper_var_templates(self, prep_temp_pep):
tp = prep_temp_pep
Expand All @@ -154,10 +157,13 @@ def test_looper_cli_pipeline(self, prep_temp_pep):
"""CLI-specified pipelines overwrite ones from config"""
tp = prep_temp_pep
pi_pth = os.path.join(os.path.dirname(tp), PIS.format("1"))
stdout, stderr, rc = subp_exec(tp, "run", ["--pipeline-interfaces", pi_pth])
print_standard_stream(stderr)
assert rc == 0
assert "Commands submitted: 3 of 3" not in str(stdout)
x = test_args_expansion(tp, "run", ["--pipeline-interfaces", pi_pth])
try:
result = main(test_args=x)
print(result)
assert result["Commands submitted"] != "Commands submitted: 3 of 3"
except Exception:
raise pytest.fail("DID RAISE {0}".format(Exception))

def test_looper_no_pipeline(self, prep_temp_pep):
"""
Expand All @@ -167,11 +173,13 @@ def test_looper_no_pipeline(self, prep_temp_pep):
tp = prep_temp_pep
with mod_yaml_data(tp) as config_data:
del config_data[SAMPLE_MODS_KEY][CONSTANT_KEY][PIPELINE_INTERFACES_KEY]
stdout, stderr, rc = subp_exec(tp, "run")
print_standard_stream(stderr)
assert rc == 0
assert "Jobs submitted: 0" in str(stderr)
assert "No pipeline interfaces defined"
x = test_args_expansion(tp, "run")
try:
result = main(test_args=x)
print(result)
assert result["Jobs submitted"] == 0
except Exception:
raise pytest.fail("DID RAISE {0}".format(Exception))

def test_looper_pipeline_not_found(self, prep_temp_pep):
"""
Expand All @@ -182,11 +190,14 @@ def test_looper_pipeline_not_found(self, prep_temp_pep):
config_data[SAMPLE_MODS_KEY][CONSTANT_KEY][PIPELINE_INTERFACES_KEY] = [
"bogus"
]
stdout, stderr, rc = subp_exec(tp, "run")
print_standard_stream(stderr)
assert rc == 0
assert "Jobs submitted: 0" in str(stderr)
assert "Ignoring invalid pipeline interface source"
x = test_args_expansion(tp, "run")
try:
result = main(test_args=x)
print(result)
assert result["Jobs submitted"] == 0
assert "No pipeline interfaces defined" in result.keys()
except Exception:
raise pytest.fail("DID RAISE {0}".format(Exception))

def test_looper_pipeline_invalid(self, prep_temp_pep):
"""
Expand All @@ -204,12 +215,14 @@ def test_looper_pipeline_invalid(self, prep_temp_pep):
piface_path = os.path.join(os.path.dirname(tp), pifaces[1])
with mod_yaml_data(piface_path) as piface_data:
del piface_data["pipeline_name"]
stdout, stderr, rc = subp_exec(tp, "run")
print_standard_stream(stderr)
assert rc == 0
assert "Jobs submitted: 0" in str(stderr)
assert "Ignoring invalid pipeline interface source"
assert "'pipeline_name' is a required property"
x = test_args_expansion(tp, "run")
try:
result = main(test_args=x)
print(result)
assert result["Jobs submitted"] == 0
assert "No pipeline interfaces defined" in result.keys()
except Exception:
raise pytest.fail("DID RAISE {0}".format(Exception))

def test_looper_sample_attr_missing(self, prep_temp_pep):
"""
Expand All @@ -218,10 +231,13 @@ def test_looper_sample_attr_missing(self, prep_temp_pep):
tp = prep_temp_pep
with mod_yaml_data(tp) as config_data:
del config_data[SAMPLE_MODS_KEY][CONSTANT_KEY]["attr"]
stdout, stderr, rc = subp_exec(tp, "run")
print_standard_stream(stderr)
assert rc == 0
assert "Jobs submitted: 0" in str(stderr)
x = test_args_expansion(tp, "run")
try:
result = main(test_args=x)
print(result)
assert result["Jobs submitted"] == 0
except Exception:
raise pytest.fail("DID RAISE {0}".format(Exception))

@pytest.mark.skipif(not is_connected(), reason="Test needs an internet connection")
def test_looper_sample_name_whitespace(self, prep_temp_pep):
Expand Down Expand Up @@ -250,10 +266,13 @@ def test_looper_toggle(self, prep_temp_pep):
tp = prep_temp_pep
with mod_yaml_data(tp) as config_data:
config_data[SAMPLE_MODS_KEY][CONSTANT_KEY][SAMPLE_TOGGLE_ATTR] = 0
stdout, stderr, rc = subp_exec(tp, "run")
print_standard_stream(stderr)
assert rc == 0
assert "Jobs submitted: 0" in str(stderr)
x = test_args_expansion(tp, "run")
try:
result = main(test_args=x)
print(result)
assert result["Jobs submitted"] == 0
except Exception:
raise pytest.fail("DID RAISE {0}".format(Exception))

@pytest.mark.parametrize("arg", CMD_STRS)
def test_cmd_extra_sample(self, prep_temp_pep, arg):
Expand Down Expand Up @@ -525,15 +544,21 @@ class TestLooperConfig:
@pytest.mark.parametrize("cmd", ["run", "runp"])
def test_init_config_file(self, prep_temp_pep, cmd, dotfile_path):
tp = prep_temp_pep
stdout, stderr, rc = subp_exec(tp, "init")
print_standard_stream(stderr)
print_standard_stream(stdout)
assert rc == 0
# stdout, stderr, rc = subp_exec(tp, "init")
# print_standard_stream(stderr)
# print_standard_stream(stdout)
x = test_args_expansion(tp, "init")
try:
result = main(test_args=x)
except Exception:
raise pytest.fail("DID RAISE {0}".format(Exception))
assert result == 0
assert_content_in_all_files(dotfile_path, tp)
stdout, stderr, rc = subp_exec(cmd=cmd)
print_standard_stream(stderr)
print_standard_stream(stdout)
assert rc == 0
x = test_args_expansion(tp, cmd)
try:
result = main(test_args=x)
except Exception:
raise pytest.fail("DID RAISE {0}".format(Exception))

def test_correct_execution_of_config(self, prepare_pep_with_dot_file):
dot_file_path = prepare_pep_with_dot_file
Expand Down

0 comments on commit dabcd65

Please sign in to comment.