diff --git a/reframe/utility/lmod_helper.py b/reframe/utility/lmod_helper.py new file mode 100644 index 0000000000..1cb05abd30 --- /dev/null +++ b/reframe/utility/lmod_helper.py @@ -0,0 +1,67 @@ +import subprocess + +class PopenCommunicate(): + def exec(self, command): + proc=subprocess.Popen(command, + shell = True, + stderr=subprocess.STDOUT, + stdout=subprocess.PIPE) + return str(proc.communicate()[0]) # None-terminated, removed None + + +def _all_avail_mods(thing_to_load, ml_av_output): + all_versions = [] + look_for = thing_to_load + "/" + for item in ml_av_output.replace("\n", " ").split(" "): + if item.strip().startswith(look_for): + all_versions.append(item.strip().split("/")[1]) + + return all_versions + +def _loop_among_one(module_glob, pc, env_context=""): + glob_to_load = module_glob.split()[-1] + if not glob_to_load.endswith("/*"): + raise Exception("Environment bit '" + module_glob + + "' is incorrect. Must be like 'module load gnu/*'") + + glob_to_load = glob_to_load[0:len(glob_to_load)-2] + + ml_av_out = pc.exec(env_context + "module avail " + glob_to_load) + all_versions = _all_avail_mods(glob_to_load, ml_av_out) + all_mods = [] + for a_version in all_versions: + all_mods.append(module_glob.replace("*", a_version)) + return all_mods + +def _expand_module_set(module_set, module_glob, pc): + new_module_set = [] + if len(module_set) == 0: + new_module_set = _loop_among_one(module_glob, pc) + else: + # combinatorial explosion of all that is found + for item in module_set: + for new_module in _loop_among_one(module_glob, pc, item + "; "): + new_module_set.append(item + "; " + new_module) + return new_module_set + +def _append_to_all(module_set, a_module): + if len(module_set) == 0: + module_set = [a_module] + else: + for i, item in enumerate(module_set): + module_set[i] = item + "; " + a_module + return module_set + +popen_comm = PopenCommunicate() +def loop_among_all(required_modules, pc=popen_comm): + star_count = required_modules.count("*") + if star_count == 0: + return [required_modules] + + module_set = [] + for module_glob in required_modules.split(";"): + if not "*" in module_glob: + module_set = _append_to_all(module_set, module_glob.strip()) + else: + module_set = _expand_module_set(module_set, module_glob.strip(), pc) + return module_set diff --git a/unittests/test_lmod_helper.py b/unittests/test_lmod_helper.py new file mode 100644 index 0000000000..72b91da6c1 --- /dev/null +++ b/unittests/test_lmod_helper.py @@ -0,0 +1,142 @@ +import pytest +import reframe.utility.lmod_helper as lmod_helper + +# On Cheyenne, output of `module reset; module rm intel; module avail gnu` +ml_av_out_easy = """ +Resetting modules to system default + +Activating Modules: + 1) ncarcompilers/0.5.0 2) netcdf/4.7.3 3) openmpi/3.1.4 + +Inactive Modules: + 1) ncarcompilers/0.5.0 2) netcdf/4.7.3 3) openmpi/3.1.4 + + +--------------------------------------------------------------------------------------------- /glade/u/apps/dav/modulefiles/default/compilers --------------------------------------------------------------------------------------------- + gnu/6.4.0 gnu/7.3.0 gnu/7.4.0 gnu/8.3.0 (D) gnu/9.1.0 + +----------------------------------------------------------------------------------------------- /glade/u/apps/dav/modulefiles/default/idep ------------------------------------------------------------------------------------------------ + gnuplot/5.2.2 + + Where: + D: Default Module + +Use "module spider" to find all possible modules. +Use "module keyword key1 key2 ..." to search for all possible modules matching any of the "keys". + +""" + +# On Cheyenne, output of `module reset; module rm intel; module load gnu/7.3.0; module avail netcdf` +ml_av_out_tricky = """ +Resetting modules to system default + +Activating Modules: + 1) ncarcompilers/0.5.0 2) netcdf/4.7.3 3) openmpi/3.1.4 + + +Inactive Modules: + 1) ncarcompilers/0.5.0 2) netcdf/4.7.3 3) openmpi/3.1.4 + + +Activating Modules: + 1) openmpi/3.1.4 + + +--------------------------------------------------------------------------------------------- /glade/u/apps/dav/modulefiles/default/gnu/7.3.0 --------------------------------------------------------------------------------------------- + netcdf/4.6.0 netcdf/4.6.1 (D) + + Where: + D: Default Module + +Use "module spider" to find all possible modules. +Use "module keyword key1 key2 ..." to search for all possible modules matching any of the "keys". + +""" + +class PopenCommunicateMock(): + def __init__(self, output, always=True): + self.output = output + self.always = always + + def exec(self, command): + if self.always: + return self.output + else: + return self.output[command] + +def test_all_avail_mods(): + expected = ['6.4.0', '7.3.0', '7.4.0', '8.3.0', '9.1.0'] + assert expected == lmod_helper._all_avail_mods("gnu", ml_av_out_easy) + +def test_all_avail_mods_tricky(): + expected = ['netcdf/4.6.0', 'netcdf/4.6.1'] + assert expected == lmod_helper._all_avail_mods("netcdf", ml_av_out_tricky) + +@pytest.mark.skip("automatically tested with _expand_module_set") +def test_loop_among_one(): + assert False + +def test_expand_module_set_simple(): + module_glob = 'module load gnu/*' + module_set = ['module reset; module rm intel'] + expected = ['module reset; module rm intel; module load gnu/6.4.0', + 'module reset; module rm intel; module load gnu/7.3.0', + 'module reset; module rm intel; module load gnu/7.4.0', + 'module reset; module rm intel; module load gnu/8.3.0', + 'module reset; module rm intel; module load gnu/9.1.0'] + pc = PopenCommunicateMock(ml_av_out_easy) + assert expected == lmod_helper._expand_module_set(module_set, module_glob, pc) + +def test_expand_module_set_nested(): + module_glob = 'module load netcdf/*' + module_set = ['module load gnu/6.4.0', + 'module load gnu/7.3.0', + 'module load gnu/7.4.0'] + + mock_rules = { # keys must match the module set + 'module load gnu/6.4.0; module avail netcdf': 'netcdf/4.7.3', + 'module load gnu/7.3.0; module avail netcdf': 'netcdf/4.7.3 netcdf/4.6.0 netcdf/4.6.1', + 'module load gnu/7.4.0; module avail netcdf': 'netcdf/4.7.1 netcdf/4.6.0 netcdf/4.6.3' } + + expected = ['module load gnu/6.4.0; module load netcdf/4.7.3', + + 'module load gnu/7.3.0; module load netcdf/4.7.3', + 'module load gnu/7.3.0; module load netcdf/4.6.0', + 'module load gnu/7.3.0; module load netcdf/4.6.1', + + 'module load gnu/7.4.0; module load netcdf/4.7.1', + 'module load gnu/7.4.0; module load netcdf/4.6.0', + 'module load gnu/7.4.0; module load netcdf/4.6.3' ] + pc = PopenCommunicateMock(mock_rules, always = False) + assert expected == lmod_helper._expand_module_set(module_set, module_glob, pc) + +def test_nothing_to_do(): + no_glob = "module reset; module sw intel gnu; module load netcdf/1.2.3; module load python/4.5.6" + results = lmod_helper.loop_among_all(no_glob) + assert results == [no_glob] + +def test_loop_among_all_in_the_middle(): + pc = PopenCommunicateMock(ml_av_out_easy) + result = lmod_helper.loop_among_all("module reset; module sw intel gnu/*; module load netcdf/1.2.3; module load python/4.5.6", pc) + expected_versions = ['6.4.0', '7.3.0', '7.4.0', '8.3.0', '9.1.0'] + expected = ["module reset; module sw intel gnu/" + i + "; module load netcdf/1.2.3; module load python/4.5.6" for i in expected_versions] + assert expected == result + +# partially tested by test_expand_module_set_nested, here checking that the things are put together correctly +def test_loop_among_all_two_globs(): + mock_rules = { 'ml reset; ml rm intel; module avail gnu': 'gnu/6.4.0 gnu/7.3.0 gnu/7.4.0', + 'ml reset; ml rm intel; ml gnu/6.4.0; module avail netcdf': 'netcdf/4.7.3', + 'ml reset; ml rm intel; ml gnu/7.3.0; module avail netcdf': 'netcdf/4.7.3 netcdf/4.6.0 netcdf/4.6.1', + 'ml reset; ml rm intel; ml gnu/7.4.0; module avail netcdf': 'netcdf/4.7.1 netcdf/4.6.0 netcdf/4.6.3' } + pc = PopenCommunicateMock(mock_rules, always = False) + results = lmod_helper.loop_among_all("ml reset; ml rm intel; ml gnu/*; ml netcdf/*", pc) + expected = ['ml reset; ml rm intel; ml gnu/6.4.0; ml netcdf/4.7.3', + + 'ml reset; ml rm intel; ml gnu/7.3.0; ml netcdf/4.7.3', + 'ml reset; ml rm intel; ml gnu/7.3.0; ml netcdf/4.6.0', + 'ml reset; ml rm intel; ml gnu/7.3.0; ml netcdf/4.6.1', + + 'ml reset; ml rm intel; ml gnu/7.4.0; ml netcdf/4.7.1', + 'ml reset; ml rm intel; ml gnu/7.4.0; ml netcdf/4.6.0', + 'ml reset; ml rm intel; ml gnu/7.4.0; ml netcdf/4.6.3' ] + assert expected == results