[wip] [feat] Module helper

reframe/utility/lmod_helper.py

@@ -0,0 +1,67 @@
+import subprocess
+
+class PopenCommunicate():
+    def exec(self, command):
+        proc=subprocess.Popen(command,
+                              shell = True,
+                              stderr=subprocess.STDOUT,
+                              stdout=subprocess.PIPE)
+        return str(proc.communicate()[0]) # None-terminated, removed None
+
+
+def _all_avail_mods(thing_to_load, ml_av_output):
+    all_versions = []
+    look_for =  thing_to_load + "/"
+    for item in ml_av_output.replace("\n", " ").split(" "):
+        if item.strip().startswith(look_for):
+            all_versions.append(item.strip().split("/")[1])
+
+    return all_versions
+
+def _loop_among_one(module_glob, pc, env_context=""):
+    glob_to_load = module_glob.split()[-1]
+    if not glob_to_load.endswith("/*"):
+        raise Exception("Environment bit '" + module_glob +
+                        "' is incorrect. Must be like 'module load gnu/*'")
+
+    glob_to_load = glob_to_load[0:len(glob_to_load)-2]
+
+    ml_av_out = pc.exec(env_context + "module avail " + glob_to_load)
+    all_versions = _all_avail_mods(glob_to_load, ml_av_out)
+    all_mods = []
+    for a_version in all_versions:
+        all_mods.append(module_glob.replace("*", a_version))
+    return all_mods
+
+def _expand_module_set(module_set, module_glob, pc):
+    new_module_set = []
+    if len(module_set) == 0:
+        new_module_set = _loop_among_one(module_glob, pc)
+    else:
+        # combinatorial explosion of all that is found
+        for item in module_set:
+            for new_module in _loop_among_one(module_glob, pc, item + "; "):
+                new_module_set.append(item + "; " + new_module)
+    return new_module_set
+
+def _append_to_all(module_set, a_module):
+    if len(module_set) == 0:
+        module_set = [a_module]
+    else:
+        for i, item in enumerate(module_set):
+            module_set[i] = item + "; " + a_module
+    return module_set
+
+popen_comm = PopenCommunicate()
+def loop_among_all(required_modules, pc=popen_comm):
+    star_count = required_modules.count("*")
+    if star_count == 0:
+        return [required_modules]
+
+    module_set = []
+    for module_glob in required_modules.split(";"):
+        if not "*" in module_glob:
+            module_set = _append_to_all(module_set, module_glob.strip())
+        else:
+            module_set = _expand_module_set(module_set, module_glob.strip(), pc)
+    return module_set

unittests/test_lmod_helper.py

@@ -0,0 +1,142 @@
+import pytest
+import reframe.utility.lmod_helper as lmod_helper
+
+# On Cheyenne, output of `module reset; module rm intel; module avail gnu`
+ml_av_out_easy = """
+Resetting modules to system default
+
+Activating Modules:
+  1) ncarcompilers/0.5.0     2) netcdf/4.7.3     3) openmpi/3.1.4
+
+Inactive Modules:
+  1) ncarcompilers/0.5.0     2) netcdf/4.7.3     3) openmpi/3.1.4
+
+
+--------------------------------------------------------------------------------------------- /glade/u/apps/dav/modulefiles/default/compilers ---------------------------------------------------------------------------------------------
+   gnu/6.4.0    gnu/7.3.0    gnu/7.4.0    gnu/8.3.0 (D)    gnu/9.1.0
+
+----------------------------------------------------------------------------------------------- /glade/u/apps/dav/modulefiles/default/idep ------------------------------------------------------------------------------------------------
+   gnuplot/5.2.2
+
+  Where:
+   D:  Default Module
+
+Use "module spider" to find all possible modules.
+Use "module keyword key1 key2 ..." to search for all possible modules matching any of the "keys".
+
+"""
+
+# On Cheyenne, output of `module reset; module rm intel; module load gnu/7.3.0; module avail netcdf`
+ml_av_out_tricky = """
+Resetting modules to system default
+
+Activating Modules:
+  1) ncarcompilers/0.5.0     2) netcdf/4.7.3     3) openmpi/3.1.4
+
+
+Inactive Modules:
+  1) ncarcompilers/0.5.0     2) netcdf/4.7.3     3) openmpi/3.1.4
+
+
+Activating Modules:
+  1) openmpi/3.1.4
+
+
+--------------------------------------------------------------------------------------------- /glade/u/apps/dav/modulefiles/default/gnu/7.3.0 ---------------------------------------------------------------------------------------------
+   netcdf/4.6.0    netcdf/4.6.1 (D)
+
+  Where:
+   D:  Default Module
+
+Use "module spider" to find all possible modules.
+Use "module keyword key1 key2 ..." to search for all possible modules matching any of the "keys".
+
+"""
+
+class PopenCommunicateMock():
+    def __init__(self, output, always=True):
+        self.output = output
+        self.always = always
+
+    def exec(self, command):
+        if self.always:
+            return self.output
+        else:
+            return self.output[command]
+
+def test_all_avail_mods():
+    expected = ['6.4.0', '7.3.0', '7.4.0', '8.3.0', '9.1.0']
+    assert expected == lmod_helper._all_avail_mods("gnu", ml_av_out_easy)
+
+def test_all_avail_mods_tricky():
+    expected = ['netcdf/4.6.0', 'netcdf/4.6.1']
+    assert expected == lmod_helper._all_avail_mods("netcdf", ml_av_out_tricky)
+
+@pytest.mark.skip("automatically tested with _expand_module_set")
+def test_loop_among_one():
+    assert False
+
+def test_expand_module_set_simple():
+    module_glob = 'module load gnu/*'
+    module_set  = ['module reset; module rm intel']
+    expected =    ['module reset; module rm intel; module load gnu/6.4.0',
+                   'module reset; module rm intel; module load gnu/7.3.0',
+                   'module reset; module rm intel; module load gnu/7.4.0',
+                   'module reset; module rm intel; module load gnu/8.3.0',
+                   'module reset; module rm intel; module load gnu/9.1.0']
+    pc = PopenCommunicateMock(ml_av_out_easy)
+    assert expected == lmod_helper._expand_module_set(module_set, module_glob, pc)
+
+def test_expand_module_set_nested():
+    module_glob = 'module load netcdf/*'
+    module_set  = ['module load gnu/6.4.0',
+                   'module load gnu/7.3.0',
+                   'module load gnu/7.4.0']
+
+    mock_rules = { # keys must match the module set
+                   'module load gnu/6.4.0; module avail netcdf': 'netcdf/4.7.3',
+                   'module load gnu/7.3.0; module avail netcdf': 'netcdf/4.7.3 netcdf/4.6.0 netcdf/4.6.1',
+                   'module load gnu/7.4.0; module avail netcdf': 'netcdf/4.7.1 netcdf/4.6.0 netcdf/4.6.3' }
+
+    expected =    ['module load gnu/6.4.0; module load netcdf/4.7.3',
+
+                   'module load gnu/7.3.0; module load netcdf/4.7.3',
+                   'module load gnu/7.3.0; module load netcdf/4.6.0',
+                   'module load gnu/7.3.0; module load netcdf/4.6.1',
+
+                   'module load gnu/7.4.0; module load netcdf/4.7.1',
+                   'module load gnu/7.4.0; module load netcdf/4.6.0',
+                   'module load gnu/7.4.0; module load netcdf/4.6.3' ]
+    pc = PopenCommunicateMock(mock_rules, always = False)
+    assert expected == lmod_helper._expand_module_set(module_set, module_glob, pc)
+
+def test_nothing_to_do():
+    no_glob = "module reset; module sw intel gnu; module load netcdf/1.2.3; module load python/4.5.6"
+    results = lmod_helper.loop_among_all(no_glob)
+    assert results == [no_glob]
+
+def test_loop_among_all_in_the_middle():
+    pc = PopenCommunicateMock(ml_av_out_easy)
+    result = lmod_helper.loop_among_all("module reset; module sw intel gnu/*; module load netcdf/1.2.3; module load python/4.5.6", pc)
+    expected_versions = ['6.4.0', '7.3.0', '7.4.0', '8.3.0', '9.1.0']
+    expected = ["module reset; module sw intel gnu/" + i + "; module load netcdf/1.2.3; module load python/4.5.6" for i in expected_versions]
+    assert expected == result
+
+# partially tested by test_expand_module_set_nested, here checking that the things are put together correctly
+def test_loop_among_all_two_globs():
+    mock_rules = { 'ml reset; ml rm intel; module avail gnu':                  'gnu/6.4.0 gnu/7.3.0 gnu/7.4.0',
+                   'ml reset; ml rm intel; ml gnu/6.4.0; module avail netcdf': 'netcdf/4.7.3',
+                   'ml reset; ml rm intel; ml gnu/7.3.0; module avail netcdf': 'netcdf/4.7.3 netcdf/4.6.0 netcdf/4.6.1',
+                   'ml reset; ml rm intel; ml gnu/7.4.0; module avail netcdf': 'netcdf/4.7.1 netcdf/4.6.0 netcdf/4.6.3' }
+    pc = PopenCommunicateMock(mock_rules, always = False)
+    results = lmod_helper.loop_among_all("ml reset; ml rm intel; ml gnu/*; ml netcdf/*", pc)
+    expected =    ['ml reset; ml rm intel; ml gnu/6.4.0; ml netcdf/4.7.3',
+
+                   'ml reset; ml rm intel; ml gnu/7.3.0; ml netcdf/4.7.3',
+                   'ml reset; ml rm intel; ml gnu/7.3.0; ml netcdf/4.6.0',
+                   'ml reset; ml rm intel; ml gnu/7.3.0; ml netcdf/4.6.1',
+
+                   'ml reset; ml rm intel; ml gnu/7.4.0; ml netcdf/4.7.1',
+                   'ml reset; ml rm intel; ml gnu/7.4.0; ml netcdf/4.6.0',
+                   'ml reset; ml rm intel; ml gnu/7.4.0; ml netcdf/4.6.3' ]
+    assert expected == results