diff --git a/config/cscs-ci.py b/config/cscs-ci.py index 32a4dc8e4a..10b132a0b4 100644 --- a/config/cscs-ci.py +++ b/config/cscs-ci.py @@ -189,8 +189,6 @@ { 'name': 'default', 'scheduler': 'local', - 'modules': [], - 'access': [], 'environs': [ 'builtin' ], diff --git a/reframe/core/modules.py b/reframe/core/modules.py index 4c64b9c825..5547f7e534 100644 --- a/reframe/core/modules.py +++ b/reframe/core/modules.py @@ -152,6 +152,14 @@ def resolve_module(self, name): def backend(self): return(self._backend) + def available_modules(self, substr=None): + '''Return a list of available modules that contain ``substr`` in their + name. + + :rtype: List[str] + ''' + return [str(m) for m in self._backend.available_modules(substr or '')] + def loaded_modules(self): '''Return a list of loaded modules. @@ -325,6 +333,13 @@ def __str__(self): class ModulesSystemImpl(abc.ABC): '''Abstract base class for module systems.''' + @abc.abstractmethod + def available_modules(self, substr): + '''Return a list of available modules, whose name contains ``substr``. + + This method returns a list of Module instances. + ''' + @abc.abstractmethod def loaded_modules(self): '''Return a list of loaded modules. @@ -470,6 +485,21 @@ def _exec_module_command(self, *args, msg=None): completed = self._run_module_command(*args, msg=msg) exec(completed.stdout) + def available_modules(self, substr): + completed = self._run_module_command( + 'avail', '-t', substr, msg='could not retrieve available modules' + ) + ret = [] + for line in completed.stderr.split('\n'): + if not line or line[-1] == ':': + # Ignore empty lines and path entries + continue + + module = re.sub(r'\(default\)', '', line) + ret.append(Module(module)) + + return ret + def loaded_modules(self): try: # LOADEDMODULES may be defined but empty @@ -681,6 +711,21 @@ def name(self): def _module_command_failed(self, completed): return completed.stdout.strip() == 'false' + def available_modules(self, substr): + completed = self._run_module_command( + '-t', 'avail', substr, msg='could not retrieve available modules' + ) + ret = [] + for line in completed.stderr.split('\n'): + if not line or line[-1] == ':': + # Ignore empty lines and path entries + continue + + module = re.sub(r'\(\S+\)', '', line) + ret.append(Module(module)) + + return ret + def conflicted_modules(self, module): completed = self._run_module_command( 'show', str(module), msg="could not show module '%s'" % module) @@ -710,6 +755,9 @@ def unload_all(self): class NoModImpl(ModulesSystemImpl): '''A convenience class that implements a no-op a modules system.''' + def available_modules(self, substr): + return [] + def loaded_modules(self): return [] diff --git a/reframe/utility/__init__.py b/reframe/utility/__init__.py index 4debc7cefa..2107857e4c 100644 --- a/reframe/utility/__init__.py +++ b/reframe/utility/__init__.py @@ -16,6 +16,7 @@ import types from collections import UserDict +from . import typecheck as typ def seconds_to_hms(seconds): @@ -265,6 +266,99 @@ def longest(*iterables): return ret +def find_modules(substr, environ_mapping=None): + '''Return all modules in the current system that contain ``substr`` in + their name. + + This function is a generator and will yield tuples of partition, + environment and module combinations for each partition of the current + system and for each environment of a partition. + + The ``environ_mapping`` argument allows you to map module name patterns to + ReFrame environments. This is useful for flat module name schemes, in + order to avoid incompatible combinations of modules and environments. + + You can use this function to parametrize regression tests over the + available environment modules. The following example will generate tests + for all the available ``netcdf`` packages in the system: + + .. code:: python + + @rfm.parameterized_test(*find_modules('netcdf')) + class MyTest(rfm.RegressionTest): + def __init__(self, s, e, m): + self.descr = f'{s}, {e}, {m}' + self.valid_systems = [s] + self.valid_prog_environs = [e] + self.modules = [m] + ... + + The following example shows the use of ``environ_mapping`` with flat + module name schemes. In this example, the toolchain for which the package + was built is encoded in the module's name. Using the ``environ_mapping`` + argument we can map module name patterns to ReFrame environments, so that + invalid combinations are pruned: + + .. code:: python + + my_find_modules = functools.partial(find_modules, environ_mapping={ + r'.*CrayGNU.*': {'PrgEnv-gnu'}, + r'.*CrayIntel.*': {'PrgEnv-intel'}, + r'.*CrayCCE.*': {'PrgEnv-cray'} + }) + + @rfm.parameterized_test(*my_find_modules('GROMACS')) + class MyTest(rfm.RegressionTest): + def __init__(self, s, e, m): + self.descr = f'{s}, {e}, {m}' + self.valid_systems = [s] + self.valid_prog_environs = [e] + self.modules = [m] + ... + + :arg substr: A substring that the returned module names must contain. + :arg environ_mapping: A dictionary mapping regular expressions to + environment names. + + :returns: An iterator that iterates over tuples of the module, partition + and environment name combinations that were found. + + ''' + + import reframe.core.runtime as rt + + if not isinstance(substr, str): + raise TypeError("'substr' argument must be a string") + + if (environ_mapping is not None and + not isinstance(environ_mapping, typ.Dict[str, str])): + raise TypeError( + "'environ_mapping' argument must be of type Dict[str,str]" + ) + + def _is_valid_for_env(m, e): + if environ_mapping is None: + return True + + for patt, environs in environ_mapping.items(): + if re.match(patt, m) and e in environs: + return True + + return False + + ms = rt.runtime().modules_system + current_system = rt.runtime().system + snap0 = rt.snapshot() + for p in current_system.partitions: + for e in p.environs: + rt.loadenv(p.local_env, e) + modules = ms.available_modules(substr) + snap0.restore() + for m in modules: + if _is_valid_for_env(m, e.name): + yield (p.fullname, e.name, m) + + class ScopedDict(UserDict): '''This is a special dict that imposes scopes on its keys. diff --git a/unittests/test_modules.py b/unittests/test_modules.py index c9c2cea51f..73af60d4aa 100644 --- a/unittests/test_modules.py +++ b/unittests/test_modules.py @@ -9,13 +9,18 @@ import reframe.core.environments as env import reframe.core.modules as modules +import reframe.utility as util import unittests.fixtures as fixtures from reframe.core.exceptions import ConfigError, EnvironError from reframe.core.runtime import runtime @pytest.fixture(params=['tmod', 'tmod4', 'lmod', 'nomod']) -def modules_system(request): +def modules_system(request, monkeypatch): + # Always pretend to be on a clean modules environment + monkeypatch.setenv('MODULEPATH', '') + monkeypatch.setenv('LOADEDMODULES', '') + monkeypatch.setenv('_LMFILES_', '') args = [request.param] if request.param != 'nomod' else [] try: m = modules.ModulesSystem.create(*args) @@ -105,6 +110,23 @@ def test_module_conflict_list(modules_system): assert 'testmod_boo' in conflict_list +def test_module_available_all(modules_system): + modules = sorted(modules_system.available_modules()) + if modules_system.name == 'nomod': + assert modules == [] + else: + assert (modules == ['testmod_bar', 'testmod_base', + 'testmod_boo', 'testmod_foo']) + + +def test_module_available_substr(modules_system): + modules = sorted(modules_system.available_modules('testmod_b')) + if modules_system.name == 'nomod': + assert modules == [] + else: + assert (modules == ['testmod_bar', 'testmod_base', 'testmod_boo']) + + @fixtures.dispatch('modules_system', suffix=lambda ms: ms.name) def test_emit_load_commands(modules_system): modules_system.module_map = { @@ -240,6 +262,9 @@ def unload_module(self, module): def is_module_loaded(self, module): return module.name in self._loaded_modules + def available_modules(self, substr): + return [] + def name(self): return 'nomod_debug' diff --git a/unittests/test_utility.py b/unittests/test_utility.py index 8465d3f9f0..68faa08020 100644 --- a/unittests/test_utility.py +++ b/unittests/test_utility.py @@ -11,11 +11,15 @@ import reframe import reframe.core.fields as fields +import reframe.core.runtime as rt import reframe.utility as util import reframe.utility.json as jsonext import reframe.utility.os_ext as os_ext import reframe.utility.sanity as sn -from reframe.core.exceptions import (SpawnedProcessError, +import unittests.fixtures as fixtures + +from reframe.core.exceptions import (ConfigError, + SpawnedProcessError, SpawnedProcessTimeout) @@ -1297,6 +1301,75 @@ def test_cray_cle_info_missing_parts(tmp_path): assert cle_info.patchset == '09' +@pytest.fixture +def temp_runtime(tmp_path): + def _temp_runtime(site_config, system=None, options={}): + options.update({'systems/prefix': tmp_path}) + with rt.temp_runtime(site_config, system, options) as ctx: + yield ctx + + yield _temp_runtime + + +@pytest.fixture(params=['tmod', 'tmod4', 'lmod', 'nomod']) +def user_exec_ctx(request, temp_runtime): + if fixtures.USER_CONFIG_FILE: + config_file, system = fixtures.USER_CONFIG_FILE, fixtures.USER_SYSTEM + else: + config_file, system = fixtures.BUILTIN_CONFIG_FILE, 'generic' + + try: + yield from temp_runtime(config_file, system, + {'systems/modules_system': request.param}) + except ConfigError as e: + pytest.skip(str(e)) + + +@pytest.fixture +def modules_system(user_exec_ctx, monkeypatch): + # Pretend to be on a clean modules environment + monkeypatch.setenv('LOADEDMODULES', '') + monkeypatch.setenv('_LMFILES_', '') + + ms = rt.runtime().system.modules_system + ms.searchpath_add(fixtures.TEST_MODULES) + return ms + + +def test_find_modules(modules_system): + found_modules = [m[2] for m in util.find_modules('testmod')] + if modules_system.name == 'nomod': + assert found_modules == [] + else: + assert found_modules == ['testmod_bar', 'testmod_base', + 'testmod_boo', 'testmod_foo'] + + +def test_find_modules_env_mapping(modules_system): + found_modules = [ + m[2] for m in util.find_modules('testmod', + environ_mapping={ + r'.*_ba.*': 'builtin', + r'testmod_foo': 'foo' + }) + ] + if modules_system.name == 'nomod': + assert found_modules == [] + else: + assert found_modules == ['testmod_bar', 'testmod_base'] + + +def test_find_modules_errors(): + with pytest.raises(TypeError): + list(util.find_modules(1)) + + with pytest.raises(TypeError): + list(util.find_modules(None)) + + with pytest.raises(TypeError): + list(util.find_modules('foo', 1)) + + def test_jsonext_dump(tmp_path): json_dump = tmp_path / 'test.json' with open(json_dump, 'w') as fp: