[feat] Add utility function to look for available environment modules

config/cscs-ci.py

@@ -189,8 +189,6 @@
                 {
                     'name': 'default',
                     'scheduler': 'local',
-                    'modules': [],
-                    'access': [],
                     'environs': [
                         'builtin'
                     ],

reframe/core/modules.py

@@ -152,6 +152,14 @@ def resolve_module(self, name):
     def backend(self):
         return(self._backend)
 
+    def available_modules(self, substr=None):
+        '''Return a list of available modules that contain ``substr`` in their
+        name.
+
+        :rtype: List[str]
+        '''
+        return [str(m) for m in self._backend.available_modules(substr or '')]
+
     def loaded_modules(self):
         '''Return a list of loaded modules.
 
@@ -325,6 +333,13 @@ def __str__(self):
 class ModulesSystemImpl(abc.ABC):
     '''Abstract base class for module systems.'''
 
+    @abc.abstractmethod
+    def available_modules(self, substr):
+        '''Return a list of available modules, whose name contains ``substr``.
+
+        This method returns a list of Module instances.
+        '''
+
     @abc.abstractmethod
     def loaded_modules(self):
         '''Return a list of loaded modules.
@@ -470,6 +485,21 @@ def _exec_module_command(self, *args, msg=None):
         completed = self._run_module_command(*args, msg=msg)
         exec(completed.stdout)
 
+    def available_modules(self, substr):
+        completed = self._run_module_command(
+            'avail', '-t', substr, msg='could not retrieve available modules'
+        )
+        ret = []
+        for line in completed.stderr.split('\n'):
+            if not line or line[-1] == ':':
+                # Ignore empty lines and path entries
+                continue
+
+            module = re.sub(r'\(default\)', '', line)
+            ret.append(Module(module))
+
+        return ret
+
     def loaded_modules(self):
         try:
             # LOADEDMODULES may be defined but empty
@@ -681,6 +711,21 @@ def name(self):
     def _module_command_failed(self, completed):
         return completed.stdout.strip() == 'false'
 
+    def available_modules(self, substr):
+        completed = self._run_module_command(
+            '-t', 'avail', substr, msg='could not retrieve available modules'
+        )
+        ret = []
+        for line in completed.stderr.split('\n'):
+            if not line or line[-1] == ':':
+                # Ignore empty lines and path entries
+                continue
+
+            module = re.sub(r'\(\S+\)', '', line)
+            ret.append(Module(module))
+
+        return ret
+
     def conflicted_modules(self, module):
         completed = self._run_module_command(
             'show', str(module), msg="could not show module '%s'" % module)
@@ -710,6 +755,9 @@ def unload_all(self):
 class NoModImpl(ModulesSystemImpl):
     '''A convenience class that implements a no-op a modules system.'''
 
+    def available_modules(self, substr):
+        return []
+
     def loaded_modules(self):
         return []
 

reframe/utility/__init__.py

@@ -16,6 +16,7 @@
 import types
 
 from collections import UserDict
+from . import typecheck as typ
 
 
 def seconds_to_hms(seconds):
@@ -265,6 +266,99 @@ def longest(*iterables):
     return ret
 
 
+def find_modules(substr, environ_mapping=None):
+    '''Return all modules in the current system that contain ``substr`` in
+    their name.
+
+    This function is a generator and will yield tuples of partition,
+    environment and module combinations for each partition of the current
+    system and for each environment of a partition.
+
+    The ``environ_mapping`` argument allows you to map module name patterns to
+    ReFrame environments. This is useful for flat module name schemes, in
+    order to avoid incompatible combinations of modules and environments.
+
+    You can use this function to parametrize regression tests over the
+    available environment modules. The following example will generate tests
+    for all the available ``netcdf`` packages in the system:
+
+    .. code:: python
+
+       @rfm.parameterized_test(*find_modules('netcdf'))
+       class MyTest(rfm.RegressionTest):
+           def __init__(self, s, e, m):
+               self.descr = f'{s}, {e}, {m}'
+               self.valid_systems = [s]
+               self.valid_prog_environs = [e]
+               self.modules = [m]
+               ...
+
+    The following example shows the use of ``environ_mapping`` with flat
+    module name schemes. In this example, the toolchain for which the package
+    was built is encoded in the module's name. Using the ``environ_mapping``
+    argument we can map module name patterns to ReFrame environments, so that
+    invalid combinations are pruned:
+
+    .. code:: python
+
+       my_find_modules = functools.partial(find_modules, environ_mapping={
+           r'.*CrayGNU.*': {'PrgEnv-gnu'},
+           r'.*CrayIntel.*': {'PrgEnv-intel'},
+           r'.*CrayCCE.*': {'PrgEnv-cray'}
+       })
+
+       @rfm.parameterized_test(*my_find_modules('GROMACS'))
+       class MyTest(rfm.RegressionTest):
+           def __init__(self, s, e, m):
+               self.descr = f'{s}, {e}, {m}'
+               self.valid_systems = [s]
+               self.valid_prog_environs = [e]
+               self.modules = [m]
+               ...
+
+    :arg substr: A substring that the returned module names must contain.
+    :arg environ_mapping: A dictionary mapping regular expressions to
+        environment names.
+
+    :returns: An iterator that iterates over tuples of the module, partition
+    and environment name combinations that were found.
+
+    '''
+
+    import reframe.core.runtime as rt
+
+    if not isinstance(substr, str):
+        raise TypeError("'substr' argument must be a string")
+
+    if (environ_mapping is not None and
+        not isinstance(environ_mapping, typ.Dict[str, str])):
+        raise TypeError(
+            "'environ_mapping' argument must be of type Dict[str,str]"
+        )
+
+    def _is_valid_for_env(m, e):
+        if environ_mapping is None:
+            return True
+
+        for patt, environs in environ_mapping.items():
+            if re.match(patt, m) and e in environs:
+                return True
+
+        return False
+
+    ms = rt.runtime().modules_system
+    current_system = rt.runtime().system
+    snap0 = rt.snapshot()
+    for p in current_system.partitions:
+        for e in p.environs:
+            rt.loadenv(p.local_env, e)
+            modules = ms.available_modules(substr)
+            snap0.restore()
+            for m in modules:
+                if _is_valid_for_env(m, e.name):
+                    yield (p.fullname, e.name, m)
+
+
 class ScopedDict(UserDict):
     '''This is a special dict that imposes scopes on its keys.
 

unittests/test_modules.py

@@ -9,13 +9,18 @@
 
 import reframe.core.environments as env
 import reframe.core.modules as modules
+import reframe.utility as util
 import unittests.fixtures as fixtures
 from reframe.core.exceptions import ConfigError, EnvironError
 from reframe.core.runtime import runtime
 
 
 @pytest.fixture(params=['tmod', 'tmod4', 'lmod', 'nomod'])
-def modules_system(request):
+def modules_system(request, monkeypatch):
+    # Always pretend to be on a clean modules environment
+    monkeypatch.setenv('MODULEPATH', '')
+    monkeypatch.setenv('LOADEDMODULES', '')
+    monkeypatch.setenv('_LMFILES_', '')
     args = [request.param] if request.param != 'nomod' else []
     try:
         m = modules.ModulesSystem.create(*args)
@@ -105,6 +110,23 @@ def test_module_conflict_list(modules_system):
         assert 'testmod_boo' in conflict_list
 
 
+def test_module_available_all(modules_system):
+    modules = sorted(modules_system.available_modules())
+    if modules_system.name == 'nomod':
+        assert modules == []
+    else:
+        assert (modules == ['testmod_bar', 'testmod_base',
+                            'testmod_boo', 'testmod_foo'])
+
+
+def test_module_available_substr(modules_system):
+    modules = sorted(modules_system.available_modules('testmod_b'))
+    if modules_system.name == 'nomod':
+        assert modules == []
+    else:
+        assert (modules == ['testmod_bar', 'testmod_base', 'testmod_boo'])
+
+
 @fixtures.dispatch('modules_system', suffix=lambda ms: ms.name)
 def test_emit_load_commands(modules_system):
     modules_system.module_map = {
@@ -240,6 +262,9 @@ def unload_module(self, module):
         def is_module_loaded(self, module):
             return module.name in self._loaded_modules
 
+        def available_modules(self, substr):
+            return []
+
         def name(self):
             return 'nomod_debug'
 

unittests/test_utility.py

@@ -11,11 +11,15 @@
 
 import reframe
 import reframe.core.fields as fields
+import reframe.core.runtime as rt
 import reframe.utility as util
 import reframe.utility.json as jsonext
 import reframe.utility.os_ext as os_ext
 import reframe.utility.sanity as sn
-from reframe.core.exceptions import (SpawnedProcessError,
+import unittests.fixtures as fixtures
+
+from reframe.core.exceptions import (ConfigError,
+                                     SpawnedProcessError,
                                      SpawnedProcessTimeout)
 
 
@@ -1297,6 +1301,75 @@ def test_cray_cle_info_missing_parts(tmp_path):
     assert cle_info.patchset == '09'
 
 
+@pytest.fixture
+def temp_runtime(tmp_path):
+    def _temp_runtime(site_config, system=None, options={}):
+        options.update({'systems/prefix': tmp_path})
+        with rt.temp_runtime(site_config, system, options) as ctx:
+            yield ctx
+
+    yield _temp_runtime
+
+
+@pytest.fixture(params=['tmod', 'tmod4', 'lmod', 'nomod'])
+def user_exec_ctx(request, temp_runtime):
+    if fixtures.USER_CONFIG_FILE:
+        config_file, system = fixtures.USER_CONFIG_FILE, fixtures.USER_SYSTEM
+    else:
+        config_file, system = fixtures.BUILTIN_CONFIG_FILE, 'generic'
+
+    try:
+        yield from temp_runtime(config_file, system,
+                                {'systems/modules_system': request.param})
+    except ConfigError as e:
+        pytest.skip(str(e))
+
+
+@pytest.fixture
+def modules_system(user_exec_ctx, monkeypatch):
+    # Pretend to be on a clean modules environment
+    monkeypatch.setenv('LOADEDMODULES', '')
+    monkeypatch.setenv('_LMFILES_', '')
+
+    ms = rt.runtime().system.modules_system
+    ms.searchpath_add(fixtures.TEST_MODULES)
+    return ms
+
+
+def test_find_modules(modules_system):
+    found_modules = [m[2] for m in util.find_modules('testmod')]
+    if modules_system.name == 'nomod':
+        assert found_modules == []
+    else:
+        assert found_modules == ['testmod_bar', 'testmod_base',
+                                 'testmod_boo', 'testmod_foo']
+
+
+def test_find_modules_env_mapping(modules_system):
+    found_modules = [
+        m[2] for m in util.find_modules('testmod',
+                                        environ_mapping={
+                                            r'.*_ba.*': 'builtin',
+                                            r'testmod_foo': 'foo'
+                                        })
+    ]
+    if modules_system.name == 'nomod':
+        assert found_modules == []
+    else:
+        assert found_modules == ['testmod_bar', 'testmod_base']
+
+
+def test_find_modules_errors():
+    with pytest.raises(TypeError):
+        list(util.find_modules(1))
+
+    with pytest.raises(TypeError):
+        list(util.find_modules(None))
+
+    with pytest.raises(TypeError):
+        list(util.find_modules('foo', 1))
+
+
 def test_jsonext_dump(tmp_path):
     json_dump = tmp_path / 'test.json'
     with open(json_dump, 'w') as fp: