From d468823ded21c4459b0936a97c922d6da9d6b844 Mon Sep 17 00:00:00 2001 From: Victor Holanda Rusu Date: Tue, 14 Jul 2020 12:41:55 +0200 Subject: [PATCH 01/15] Add module available to modules interface --- reframe/core/modules.py | 39 +++++++++++++++++++++++++++++++++++++++ unittests/test_modules.py | 14 ++++++++++++++ 2 files changed, 53 insertions(+) diff --git a/reframe/core/modules.py b/reframe/core/modules.py index 4c64b9c825..e12e9dc75b 100644 --- a/reframe/core/modules.py +++ b/reframe/core/modules.py @@ -152,6 +152,13 @@ def resolve_module(self, name): def backend(self): return(self._backend) + def available_modules(self): + '''Return a list of available modules. + + :rtype: List[str] + ''' + return [str(m) for m in self._backend.available_modules()] + def loaded_modules(self): '''Return a list of loaded modules. @@ -325,6 +332,13 @@ def __str__(self): class ModulesSystemImpl(abc.ABC): '''Abstract base class for module systems.''' + @abc.abstractmethod + def available_modules(self): + '''Return a list of available modules. + + This method returns a list of Module instances. + ''' + @abc.abstractmethod def loaded_modules(self): '''Return a list of loaded modules. @@ -470,6 +484,17 @@ def _exec_module_command(self, *args, msg=None): completed = self._run_module_command(*args, msg=msg) exec(completed.stdout) + def available_modules(self): + avail = [] + completed = self._run_module_command( + 'available', '-t', msg="could not run module available") + + for line in re.finditer(r'\S+[^:]$', completed.stderr, re.MULTILINE): + module = re.sub(r"\(default\)", "", line.group(0)) + avail += [Module(module)] + + return avail + def loaded_modules(self): try: # LOADEDMODULES may be defined but empty @@ -681,6 +706,17 @@ def name(self): def _module_command_failed(self, completed): return completed.stdout.strip() == 'false' + def available_modules(self): + avail = [] + completed = self._run_module_command( + '-t', 'available', msg="could not run module available") + + for line in re.finditer(r'\S+[^:/]$', completed.stderr, re.MULTILINE): + module = re.sub(r"\(\S+\)", "", line.group(0)) + avail += [Module(module)] + + return avail + def conflicted_modules(self, module): completed = self._run_module_command( 'show', str(module), msg="could not show module '%s'" % module) @@ -710,6 +746,9 @@ def unload_all(self): class NoModImpl(ModulesSystemImpl): '''A convenience class that implements a no-op a modules system.''' + def available_modules(self): + return [] + def loaded_modules(self): return [] diff --git a/unittests/test_modules.py b/unittests/test_modules.py index b909baff03..82eb7f36b8 100644 --- a/unittests/test_modules.py +++ b/unittests/test_modules.py @@ -30,6 +30,12 @@ def test_searchpath(self): self.modules_system.searchpath_remove(TEST_MODULES) assert TEST_MODULES not in self.modules_system.searchpath + def test_module_available(self): + available_modules = self.modules_system.available_modules() + + assert 'foo' not in available_modules + assert 'testmod_foo' in available_modules + def test_module_load(self): with pytest.raises(EnvironError): self.modules_system.load_module('foo') @@ -175,6 +181,11 @@ def test_searchpath(self): # Simply test that no exceptions are thrown self.modules_system.searchpath_remove(TEST_MODULES) + def test_module_available(self): + available_modules = self.modules_system.available_modules() + + assert 0 == len(available_modules) + def test_module_load(self): self.modules_system.load_module('foo') self.modules_system.unload_module('foo') @@ -236,6 +247,9 @@ def __init__(self): self.load_seq = [] self.unload_seq = [] + def available_modules(self): + return [] + def loaded_modules(self): return list(self._loaded_modules) From e95790976d9b2f3a8cc67a1750e58aff547cd52a Mon Sep 17 00:00:00 2001 From: victorusu Date: Wed, 15 Jul 2020 13:16:08 +0200 Subject: [PATCH 02/15] Update reframe/core/modules.py Co-authored-by: Vasileios Karakasis --- reframe/core/modules.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/reframe/core/modules.py b/reframe/core/modules.py index e12e9dc75b..e7d7ea2b64 100644 --- a/reframe/core/modules.py +++ b/reframe/core/modules.py @@ -490,8 +490,8 @@ def available_modules(self): 'available', '-t', msg="could not run module available") for line in re.finditer(r'\S+[^:]$', completed.stderr, re.MULTILINE): - module = re.sub(r"\(default\)", "", line.group(0)) - avail += [Module(module)] + module = re.sub(r'\(default\)', '', line) + avail.append(Module(module)) return avail From 3a024b913aeb8168f4094a84716b10ce3b417d14 Mon Sep 17 00:00:00 2001 From: victorusu Date: Wed, 15 Jul 2020 13:16:15 +0200 Subject: [PATCH 03/15] Update reframe/core/modules.py Co-authored-by: Vasileios Karakasis --- reframe/core/modules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reframe/core/modules.py b/reframe/core/modules.py index e7d7ea2b64..693f5aebde 100644 --- a/reframe/core/modules.py +++ b/reframe/core/modules.py @@ -487,7 +487,7 @@ def _exec_module_command(self, *args, msg=None): def available_modules(self): avail = [] completed = self._run_module_command( - 'available', '-t', msg="could not run module available") + 'avail', '-t', msg="could not run 'module avail'") for line in re.finditer(r'\S+[^:]$', completed.stderr, re.MULTILINE): module = re.sub(r'\(default\)', '', line) From 3dd68cd5eafb3c55ff9169905cb380404a65252a Mon Sep 17 00:00:00 2001 From: victorusu Date: Sat, 5 Sep 2020 00:31:50 +0200 Subject: [PATCH 04/15] Address PR remarks --- reframe/core/modules.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/reframe/core/modules.py b/reframe/core/modules.py index 693f5aebde..9d042e0f90 100644 --- a/reframe/core/modules.py +++ b/reframe/core/modules.py @@ -152,12 +152,12 @@ def resolve_module(self, name): def backend(self): return(self._backend) - def available_modules(self): + def available_modules(self, module): '''Return a list of available modules. :rtype: List[str] ''' - return [str(m) for m in self._backend.available_modules()] + return [str(m) for m in self._backend.available_modules(modu, modulele)] def loaded_modules(self): '''Return a list of loaded modules. @@ -333,7 +333,7 @@ class ModulesSystemImpl(abc.ABC): '''Abstract base class for module systems.''' @abc.abstractmethod - def available_modules(self): + def available_modules(self, module): '''Return a list of available modules. This method returns a list of Module instances. @@ -484,13 +484,13 @@ def _exec_module_command(self, *args, msg=None): completed = self._run_module_command(*args, msg=msg) exec(completed.stdout) - def available_modules(self): + def available_modules(self, module, module): avail = [] completed = self._run_module_command( - 'avail', '-t', msg="could not run 'module avail'") + 'avail', '-t', module, msg="could not run 'module avail'") for line in re.finditer(r'\S+[^:]$', completed.stderr, re.MULTILINE): - module = re.sub(r'\(default\)', '', line) + module = re.sub(r'\(default\)', '', line.group(0)) avail.append(Module(module)) return avail @@ -706,10 +706,10 @@ def name(self): def _module_command_failed(self, completed): return completed.stdout.strip() == 'false' - def available_modules(self): + def available_modules(self, module): avail = [] completed = self._run_module_command( - '-t', 'available', msg="could not run module available") + '-t', 'available', module, msg="could not run module available") for line in re.finditer(r'\S+[^:/]$', completed.stderr, re.MULTILINE): module = re.sub(r"\(\S+\)", "", line.group(0)) From c3392c88f273676d530b1eb681769daf9b3efda7 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Mon, 7 Sep 2020 20:25:10 +0200 Subject: [PATCH 05/15] Fix module avail implementation --- reframe/core/modules.py | 37 +++++++++++++++++++------------------ 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/reframe/core/modules.py b/reframe/core/modules.py index 9d042e0f90..d277560d31 100644 --- a/reframe/core/modules.py +++ b/reframe/core/modules.py @@ -152,12 +152,13 @@ def resolve_module(self, name): def backend(self): return(self._backend) - def available_modules(self, module): - '''Return a list of available modules. + def available_modules(self, substr=None): + '''Return a list of available modules that contain ``substr`` in their + name. :rtype: List[str] ''' - return [str(m) for m in self._backend.available_modules(modu, modulele)] + return [str(m) for m in self._backend.available_modules(substr or '')] def loaded_modules(self): '''Return a list of loaded modules. @@ -333,8 +334,8 @@ class ModulesSystemImpl(abc.ABC): '''Abstract base class for module systems.''' @abc.abstractmethod - def available_modules(self, module): - '''Return a list of available modules. + def available_modules(self, substr): + '''Return a list of available modules, whose name contains ``substr``. This method returns a list of Module instances. ''' @@ -484,16 +485,16 @@ def _exec_module_command(self, *args, msg=None): completed = self._run_module_command(*args, msg=msg) exec(completed.stdout) - def available_modules(self, module, module): - avail = [] + def available_modules(self, substr): completed = self._run_module_command( - 'avail', '-t', module, msg="could not run 'module avail'") - + 'avail', '-t', substr, msg='could not retrieve available modules' + ) + ret = [] for line in re.finditer(r'\S+[^:]$', completed.stderr, re.MULTILINE): module = re.sub(r'\(default\)', '', line.group(0)) - avail.append(Module(module)) + ret.append(Module(module)) - return avail + return ret def loaded_modules(self): try: @@ -706,16 +707,16 @@ def name(self): def _module_command_failed(self, completed): return completed.stdout.strip() == 'false' - def available_modules(self, module): - avail = [] + def available_modules(self, substr): completed = self._run_module_command( - '-t', 'available', module, msg="could not run module available") - + '-t', 'avail', substr, msg='could not retrieve available modules' + ) + ret = [] for line in re.finditer(r'\S+[^:/]$', completed.stderr, re.MULTILINE): module = re.sub(r"\(\S+\)", "", line.group(0)) - avail += [Module(module)] + ret.append(Module(module)) - return avail + return ret def conflicted_modules(self, module): completed = self._run_module_command( @@ -746,7 +747,7 @@ def unload_all(self): class NoModImpl(ModulesSystemImpl): '''A convenience class that implements a no-op a modules system.''' - def available_modules(self): + def available_modules(self, substr): return [] def loaded_modules(self): From 58c48b854509b541d92fbd436f56137b3fa834d1 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Mon, 7 Sep 2020 21:30:20 +0200 Subject: [PATCH 06/15] WIP: Add find_modules() utility function --- reframe/utility/__init__.py | 24 ++++++++++++++++++++++++ unittests/test_modules.py | 20 +++++++++++++++++++- 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/reframe/utility/__init__.py b/reframe/utility/__init__.py index 87ba4a2c94..463684eca8 100644 --- a/reframe/utility/__init__.py +++ b/reframe/utility/__init__.py @@ -258,6 +258,30 @@ def longest(*iterables): return ret +def find_modules(module, toolchain_mapping=None): + def _is_valid_for_env(m, e): + if toolchain_mapping is None: + return True + + for patt, toolchains in toolchain_mapping.items(): + if re.match(patt, m) and e in toolchains: + return True + + return False + + ms = rt.runtime().modules_system + current_system = rt.runtime().system + snap0 = rt.snapshot() + for p in current_system.partitions: + for e in p.environs: + rt.loadenv(p.local_env, e) + modules = ms.available_modules(module) + snap0.restore() + for m in modules: + if _is_valid_for_env(m, e.name): + yield (p.fullname, e.name, m) + + class ScopedDict(UserDict): '''This is a special dict that imposes scopes on its keys. diff --git a/unittests/test_modules.py b/unittests/test_modules.py index c9c2cea51f..9f6513e06e 100644 --- a/unittests/test_modules.py +++ b/unittests/test_modules.py @@ -15,7 +15,8 @@ @pytest.fixture(params=['tmod', 'tmod4', 'lmod', 'nomod']) -def modules_system(request): +def modules_system(request, monkeypatch): + monkeypatch.setenv('MODULEPATH', '') args = [request.param] if request.param != 'nomod' else [] try: m = modules.ModulesSystem.create(*args) @@ -105,6 +106,23 @@ def test_module_conflict_list(modules_system): assert 'testmod_boo' in conflict_list +def test_module_available_all(modules_system): + modules = sorted(modules_system.available_modules()) + if modules_system.name == 'nomod': + assert modules == [] + else: + assert (modules == ['testmod_bar', 'testmod_base', + 'testmod_boo', 'testmod_foo']) + + +def test_module_available_substr(modules_system): + modules = sorted(modules_system.available_modules('testmod_b')) + if modules_system.name == 'nomod': + assert modules == [] + else: + assert (modules == ['testmod_bar', 'testmod_base', 'testmod_boo']) + + @fixtures.dispatch('modules_system', suffix=lambda ms: ms.name) def test_emit_load_commands(modules_system): modules_system.module_map = { From 1367e6de632b3560d727db853b94cd9bfec0aaef Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Mon, 7 Sep 2020 23:41:50 +0200 Subject: [PATCH 07/15] Add missing import --- reframe/utility/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/reframe/utility/__init__.py b/reframe/utility/__init__.py index 463684eca8..9a42c7204c 100644 --- a/reframe/utility/__init__.py +++ b/reframe/utility/__init__.py @@ -259,6 +259,8 @@ def longest(*iterables): def find_modules(module, toolchain_mapping=None): + import reframe.core.runtime as rt + def _is_valid_for_env(m, e): if toolchain_mapping is None: return True From 5b93ec748f9f48703ea0db13263fb0f9f7260db7 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Mon, 7 Sep 2020 23:48:09 +0200 Subject: [PATCH 08/15] Fix unit tests --- unittests/test_modules.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/unittests/test_modules.py b/unittests/test_modules.py index 9f6513e06e..85ddada397 100644 --- a/unittests/test_modules.py +++ b/unittests/test_modules.py @@ -258,6 +258,9 @@ def unload_module(self, module): def is_module_loaded(self, module): return module.name in self._loaded_modules + def available_modules(self, substr): + return [] + def name(self): return 'nomod_debug' From 4b609703d04b93a4f6f5b3eb92c390c106170653 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Tue, 8 Sep 2020 00:10:32 +0200 Subject: [PATCH 09/15] Fix Tmod unit tests --- unittests/test_modules.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/unittests/test_modules.py b/unittests/test_modules.py index 85ddada397..fcf819bc60 100644 --- a/unittests/test_modules.py +++ b/unittests/test_modules.py @@ -16,7 +16,10 @@ @pytest.fixture(params=['tmod', 'tmod4', 'lmod', 'nomod']) def modules_system(request, monkeypatch): + # Always pretend to be on a clean modules environment monkeypatch.setenv('MODULEPATH', '') + monkeypatch.setenv('LOADEDMODULES', '') + monkeypatch.setenv('_LMFILES_', '') args = [request.param] if request.param != 'nomod' else [] try: m = modules.ModulesSystem.create(*args) From c8de2fd7fe26452e19bbc3d72ab71ea2d8e16a88 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Tue, 8 Sep 2020 10:25:42 +0200 Subject: [PATCH 10/15] Fix implementation of available_modules() --- reframe/core/modules.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/reframe/core/modules.py b/reframe/core/modules.py index d277560d31..5547f7e534 100644 --- a/reframe/core/modules.py +++ b/reframe/core/modules.py @@ -490,8 +490,12 @@ def available_modules(self, substr): 'avail', '-t', substr, msg='could not retrieve available modules' ) ret = [] - for line in re.finditer(r'\S+[^:]$', completed.stderr, re.MULTILINE): - module = re.sub(r'\(default\)', '', line.group(0)) + for line in completed.stderr.split('\n'): + if not line or line[-1] == ':': + # Ignore empty lines and path entries + continue + + module = re.sub(r'\(default\)', '', line) ret.append(Module(module)) return ret @@ -712,8 +716,12 @@ def available_modules(self, substr): '-t', 'avail', substr, msg='could not retrieve available modules' ) ret = [] - for line in re.finditer(r'\S+[^:/]$', completed.stderr, re.MULTILINE): - module = re.sub(r"\(\S+\)", "", line.group(0)) + for line in completed.stderr.split('\n'): + if not line or line[-1] == ':': + # Ignore empty lines and path entries + continue + + module = re.sub(r'\(\S+\)', '', line) ret.append(Module(module)) return ret From 98c0f2f1e1b995a29c019c63d003fbb48e7c036b Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Wed, 9 Sep 2020 18:09:27 +0200 Subject: [PATCH 11/15] Add unit tests for find_module() --- reframe/utility/__init__.py | 1 + unittests/test_modules.py | 1 + unittests/test_utility.py | 72 ++++++++++++++++++++++++++++++++++++- 3 files changed, 73 insertions(+), 1 deletion(-) diff --git a/reframe/utility/__init__.py b/reframe/utility/__init__.py index 9a42c7204c..f50417c740 100644 --- a/reframe/utility/__init__.py +++ b/reframe/utility/__init__.py @@ -281,6 +281,7 @@ def _is_valid_for_env(m, e): snap0.restore() for m in modules: if _is_valid_for_env(m, e.name): + print(p.fullname, e.name, m) yield (p.fullname, e.name, m) diff --git a/unittests/test_modules.py b/unittests/test_modules.py index fcf819bc60..73af60d4aa 100644 --- a/unittests/test_modules.py +++ b/unittests/test_modules.py @@ -9,6 +9,7 @@ import reframe.core.environments as env import reframe.core.modules as modules +import reframe.utility as util import unittests.fixtures as fixtures from reframe.core.exceptions import ConfigError, EnvironError from reframe.core.runtime import runtime diff --git a/unittests/test_utility.py b/unittests/test_utility.py index 4cc1329334..f4ad803395 100644 --- a/unittests/test_utility.py +++ b/unittests/test_utility.py @@ -11,9 +11,13 @@ import reframe import reframe.core.fields as fields +import reframe.core.runtime as rt import reframe.utility as util import reframe.utility.os_ext as os_ext -from reframe.core.exceptions import (SpawnedProcessError, +import unittests.fixtures as fixtures + +from reframe.core.exceptions import (ConfigError, + SpawnedProcessError, SpawnedProcessTimeout) @@ -1293,3 +1297,69 @@ def test_cray_cle_info_missing_parts(tmp_path): assert cle_info.date is None assert cle_info.network is None assert cle_info.patchset == '09' + + +@pytest.fixture +def temp_runtime(tmp_path): + def _temp_runtime(site_config, system=None, options={}): + options.update({'systems/prefix': tmp_path}) + with rt.temp_runtime(site_config, system, options): + yield rt.runtime + + yield _temp_runtime + + +@pytest.fixture(params=['tmod', 'tmod4', 'lmod', 'nomod']) +def runtime_with_modules(request, temp_runtime): + if fixtures.USER_CONFIG_FILE: + config_file, system = fixtures.USER_CONFIG_FILE, fixtures.USER_SYSTEM + else: + config_file, system = fixtures.BUILTIN_CONFIG_FILE, 'generic' + + try: + yield from temp_runtime(config_file, system, + {'systems/modules_system': request.param}) + except ConfigError as e: + pytest.skip(str(e)) + + +def test_find_modules(monkeypatch, runtime_with_modules): + # Pretend to be on a clean modules environment + monkeypatch.setenv('MODULEPATH', '') + monkeypatch.setenv('LOADEDMODULES', '') + monkeypatch.setenv('_LMFILES_', '') + + ms = rt.runtime().system.modules_system + ms.searchpath_add(fixtures.TEST_MODULES) + found_modules = list(util.find_modules('testmod')) + if ms.name == 'nomod': + assert found_modules == [] + else: + assert found_modules == [ + ('generic:default', 'builtin', 'testmod_bar'), + ('generic:default', 'builtin', 'testmod_base'), + ('generic:default', 'builtin', 'testmod_boo'), + ('generic:default', 'builtin', 'testmod_foo') + ] + + +def test_find_modules_toolchains(monkeypatch, runtime_with_modules): + # Pretend to be on a clean modules environment + monkeypatch.setenv('MODULEPATH', '') + monkeypatch.setenv('LOADEDMODULES', '') + monkeypatch.setenv('_LMFILES_', '') + + ms = rt.runtime().system.modules_system + ms.searchpath_add(fixtures.TEST_MODULES) + found_modules = list( + util.find_modules('testmod', + toolchain_mapping={r'.*_ba.*': 'builtin', + r'testmod_foo': 'foo'}) + ) + if ms.name == 'nomod': + assert found_modules == [] + else: + assert found_modules == [ + ('generic:default', 'builtin', 'testmod_bar'), + ('generic:default', 'builtin', 'testmod_base') + ] From 96db39cdb6b807c627003a175d1ce68ae5219210 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Wed, 9 Sep 2020 18:49:24 +0200 Subject: [PATCH 12/15] Add unit tests for the find_modules() utility function --- reframe/utility/__init__.py | 8 ++++- unittests/test_utility.py | 61 ++++++++++++++++++++++++------------- 2 files changed, 47 insertions(+), 22 deletions(-) diff --git a/reframe/utility/__init__.py b/reframe/utility/__init__.py index f50417c740..ef5f76f7cc 100644 --- a/reframe/utility/__init__.py +++ b/reframe/utility/__init__.py @@ -261,6 +261,13 @@ def longest(*iterables): def find_modules(module, toolchain_mapping=None): import reframe.core.runtime as rt + if not isinstance(module, str): + raise TypeError("'module' argument must be a string") + + if (toolchain_mapping is not None and + not isinstance(toolchain_mapping, collections.abc.Mapping)): + raise TypeError("'toolchain_mapping' argument must be a mapping type") + def _is_valid_for_env(m, e): if toolchain_mapping is None: return True @@ -281,7 +288,6 @@ def _is_valid_for_env(m, e): snap0.restore() for m in modules: if _is_valid_for_env(m, e.name): - print(p.fullname, e.name, m) yield (p.fullname, e.name, m) diff --git a/unittests/test_utility.py b/unittests/test_utility.py index f4ad803395..dab09584d6 100644 --- a/unittests/test_utility.py +++ b/unittests/test_utility.py @@ -1310,29 +1310,31 @@ def _temp_runtime(site_config, system=None, options={}): @pytest.fixture(params=['tmod', 'tmod4', 'lmod', 'nomod']) -def runtime_with_modules(request, temp_runtime): +def modules_system(request, temp_runtime, monkeypatch): + # Pretend to be on a clean modules environment + monkeypatch.setenv('MODULEPATH', '') + monkeypatch.setenv('LOADEDMODULES', '') + monkeypatch.setenv('_LMFILES_', '') + if fixtures.USER_CONFIG_FILE: config_file, system = fixtures.USER_CONFIG_FILE, fixtures.USER_SYSTEM else: config_file, system = fixtures.BUILTIN_CONFIG_FILE, 'generic' try: - yield from temp_runtime(config_file, system, - {'systems/modules_system': request.param}) + next(temp_runtime(config_file, system, + {'systems/modules_system': request.param})) except ConfigError as e: pytest.skip(str(e)) + else: + ms = rt.runtime().system.modules_system + ms.searchpath_add(fixtures.TEST_MODULES) + return ms -def test_find_modules(monkeypatch, runtime_with_modules): - # Pretend to be on a clean modules environment - monkeypatch.setenv('MODULEPATH', '') - monkeypatch.setenv('LOADEDMODULES', '') - monkeypatch.setenv('_LMFILES_', '') - - ms = rt.runtime().system.modules_system - ms.searchpath_add(fixtures.TEST_MODULES) +def test_find_modules(modules_system): found_modules = list(util.find_modules('testmod')) - if ms.name == 'nomod': + if modules_system.name == 'nomod': assert found_modules == [] else: assert found_modules == [ @@ -1343,23 +1345,40 @@ def test_find_modules(monkeypatch, runtime_with_modules): ] -def test_find_modules_toolchains(monkeypatch, runtime_with_modules): - # Pretend to be on a clean modules environment - monkeypatch.setenv('MODULEPATH', '') - monkeypatch.setenv('LOADEDMODULES', '') - monkeypatch.setenv('_LMFILES_', '') - - ms = rt.runtime().system.modules_system - ms.searchpath_add(fixtures.TEST_MODULES) +def test_find_modules_toolchains(modules_system): found_modules = list( util.find_modules('testmod', toolchain_mapping={r'.*_ba.*': 'builtin', r'testmod_foo': 'foo'}) ) - if ms.name == 'nomod': + if modules_system.name == 'nomod': assert found_modules == [] else: assert found_modules == [ ('generic:default', 'builtin', 'testmod_bar'), ('generic:default', 'builtin', 'testmod_base') ] + + +def test_find_modules_all(modules_system): + found_modules = list(util.find_modules('')) + if modules_system.name == 'nomod': + assert found_modules == [] + else: + assert found_modules == [ + ('generic:default', 'builtin', 'testmod_bar'), + ('generic:default', 'builtin', 'testmod_base'), + ('generic:default', 'builtin', 'testmod_boo'), + ('generic:default', 'builtin', 'testmod_foo') + ] + + +def test_find_modules_errors(): + with pytest.raises(TypeError): + list(util.find_modules(1)) + + with pytest.raises(TypeError): + list(util.find_modules(None)) + + with pytest.raises(TypeError): + list(util.find_modules('foo', 1)) From d1cf5d24f98ee47fab3845cb562e20a15f0e1dc8 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Fri, 18 Sep 2020 19:59:45 +0200 Subject: [PATCH 13/15] Fix find_modules() unit tests --- config/cscs-ci.py | 2 -- unittests/test_utility.py | 71 +++++++++++++++------------------------ 2 files changed, 28 insertions(+), 45 deletions(-) diff --git a/config/cscs-ci.py b/config/cscs-ci.py index 32a4dc8e4a..10b132a0b4 100644 --- a/config/cscs-ci.py +++ b/config/cscs-ci.py @@ -189,8 +189,6 @@ { 'name': 'default', 'scheduler': 'local', - 'modules': [], - 'access': [], 'environs': [ 'builtin' ], diff --git a/unittests/test_utility.py b/unittests/test_utility.py index dab09584d6..97af5d8a4c 100644 --- a/unittests/test_utility.py +++ b/unittests/test_utility.py @@ -1303,74 +1303,59 @@ def test_cray_cle_info_missing_parts(tmp_path): def temp_runtime(tmp_path): def _temp_runtime(site_config, system=None, options={}): options.update({'systems/prefix': tmp_path}) - with rt.temp_runtime(site_config, system, options): - yield rt.runtime + with rt.temp_runtime(site_config, system, options) as ctx: + yield ctx yield _temp_runtime @pytest.fixture(params=['tmod', 'tmod4', 'lmod', 'nomod']) -def modules_system(request, temp_runtime, monkeypatch): - # Pretend to be on a clean modules environment - monkeypatch.setenv('MODULEPATH', '') - monkeypatch.setenv('LOADEDMODULES', '') - monkeypatch.setenv('_LMFILES_', '') - +def user_exec_ctx(request, temp_runtime): if fixtures.USER_CONFIG_FILE: config_file, system = fixtures.USER_CONFIG_FILE, fixtures.USER_SYSTEM else: config_file, system = fixtures.BUILTIN_CONFIG_FILE, 'generic' try: - next(temp_runtime(config_file, system, - {'systems/modules_system': request.param})) + yield from temp_runtime(config_file, system, + {'systems/modules_system': request.param}) except ConfigError as e: pytest.skip(str(e)) - else: - ms = rt.runtime().system.modules_system - ms.searchpath_add(fixtures.TEST_MODULES) - return ms -def test_find_modules(modules_system): - found_modules = list(util.find_modules('testmod')) - if modules_system.name == 'nomod': - assert found_modules == [] - else: - assert found_modules == [ - ('generic:default', 'builtin', 'testmod_bar'), - ('generic:default', 'builtin', 'testmod_base'), - ('generic:default', 'builtin', 'testmod_boo'), - ('generic:default', 'builtin', 'testmod_foo') - ] +@pytest.fixture +def modules_system(user_exec_ctx, monkeypatch): + # Pretend to be on a clean modules environment + # monkeypatch.setenv('MODULEPATH', '') + monkeypatch.setenv('LOADEDMODULES', '') + monkeypatch.setenv('_LMFILES_', '') + ms = rt.runtime().system.modules_system + ms.searchpath_add(fixtures.TEST_MODULES) + return ms -def test_find_modules_toolchains(modules_system): - found_modules = list( - util.find_modules('testmod', - toolchain_mapping={r'.*_ba.*': 'builtin', - r'testmod_foo': 'foo'}) - ) + +def test_find_modules(modules_system): + found_modules = [m[2] for m in util.find_modules('testmod')] if modules_system.name == 'nomod': assert found_modules == [] else: - assert found_modules == [ - ('generic:default', 'builtin', 'testmod_bar'), - ('generic:default', 'builtin', 'testmod_base') - ] + assert found_modules == ['testmod_bar', 'testmod_base', + 'testmod_boo', 'testmod_foo'] -def test_find_modules_all(modules_system): - found_modules = list(util.find_modules('')) +def test_find_modules_toolchains(modules_system): + found_modules = [ + m[2] for m in util.find_modules('testmod', + toolchain_mapping={ + r'.*_ba.*': 'builtin', + r'testmod_foo': 'foo' + }) + ] if modules_system.name == 'nomod': assert found_modules == [] else: - assert found_modules == [ - ('generic:default', 'builtin', 'testmod_bar'), - ('generic:default', 'builtin', 'testmod_base'), - ('generic:default', 'builtin', 'testmod_boo'), - ('generic:default', 'builtin', 'testmod_foo') - ] + assert found_modules == ['testmod_bar', 'testmod_base'] def test_find_modules_errors(): From cd68e0f30b4ccf1e3de45fe22c998a24ade32bd9 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Fri, 18 Sep 2020 22:15:34 +0200 Subject: [PATCH 14/15] Document the find_module() utility function Also: - Improve argument names. - Improve type checking of the `environ_mapping` argument. --- reframe/utility/__init__.py | 81 ++++++++++++++++++++++++++++++++----- unittests/test_utility.py | 4 +- 2 files changed, 73 insertions(+), 12 deletions(-) diff --git a/reframe/utility/__init__.py b/reframe/utility/__init__.py index ef5f76f7cc..ea840e0451 100644 --- a/reframe/utility/__init__.py +++ b/reframe/utility/__init__.py @@ -16,6 +16,7 @@ import types from collections import UserDict +from . import typecheck as typ def seconds_to_hms(seconds): @@ -258,22 +259,82 @@ def longest(*iterables): return ret -def find_modules(module, toolchain_mapping=None): +def find_modules(substr, environ_mapping=None): + '''Return all modules in the current system that contain ``substr`` in + their name. + + This function is a generator and will yield tuples of partition, + environment and module combinations for each partition of the current + system and for each environment of a partition. + + The ``environ_mapping`` argument allows you to map module name patterns to + ReFrame environments. This is useful for flat module name schemes, in + order to avoid incompatible combinations of modules and environments. + + You can use this function to parametrize regression tests over the + available environment modules. The following example will generate tests + for all the available ``netcdf`` packages in the system: + + .. code:: python + + @rfm.parameterized_test(*find_modules('netcdf')) + class MyTest(rfm.RegressionTest): + def __init__(self, s, e, m): + self.descr = f'{s}, {e}, {m}' + self.valid_systems = [s] + self.valid_prog_environs = [e] + self.modules = [m] + ... + + The following example shows the use of ``environ_mapping`` with flat + module name schemes. In this example, the toolchain for which the package + was built is encoded in the module's name. Using the ``environ_mapping`` + argument we can map module name patterns to ReFrame environments, so that + invalid combinations are pruned: + + .. code:: python + + my_find_modules = functools.partial(find_modules, environ_mapping={ + r'.*CrayGNU.*': {'PrgEnv-gnu'}, + r'.*CrayIntel.*': {'PrgEnv-intel'}, + r'.*CrayCCE.*': {'PrgEnv-cray'} + }) + + @rfm.parameterized_test(*my_find_modules('GROMACS')) + class MyTest(rfm.RegressionTest): + def __init__(self, s, e, m): + self.descr = f'{s}, {e}, {m}' + self.valid_systems = [s] + self.valid_prog_environs = [e] + self.modules = [m] + ... + + :arg substr: A substring that the returned module names must contain. + :arg environ_mapping: A dictionary mapping regular expressions to + environment names. + + :returns: An iterator that iterates over tuples of the module, partition + and environment name combinations that were found. + + ''' + import reframe.core.runtime as rt - if not isinstance(module, str): - raise TypeError("'module' argument must be a string") + if not isinstance(substr, str): + raise TypeError("'substr' argument must be a string") - if (toolchain_mapping is not None and - not isinstance(toolchain_mapping, collections.abc.Mapping)): - raise TypeError("'toolchain_mapping' argument must be a mapping type") + if (environ_mapping is not None and + not isinstance(environ_mapping, typ.Dict[str, str])): + raise TypeError( + "'environ_mapping' argument must be of type Dict[str,str]" + ) def _is_valid_for_env(m, e): - if toolchain_mapping is None: + if environ_mapping is None: return True - for patt, toolchains in toolchain_mapping.items(): - if re.match(patt, m) and e in toolchains: + for patt, environs in environ_mapping.items(): + if re.match(patt, m) and e in environs: return True return False @@ -284,7 +345,7 @@ def _is_valid_for_env(m, e): for p in current_system.partitions: for e in p.environs: rt.loadenv(p.local_env, e) - modules = ms.available_modules(module) + modules = ms.available_modules(substr) snap0.restore() for m in modules: if _is_valid_for_env(m, e.name): diff --git a/unittests/test_utility.py b/unittests/test_utility.py index 97af5d8a4c..ff90b54f4d 100644 --- a/unittests/test_utility.py +++ b/unittests/test_utility.py @@ -1344,10 +1344,10 @@ def test_find_modules(modules_system): 'testmod_boo', 'testmod_foo'] -def test_find_modules_toolchains(modules_system): +def test_find_modules_env_mapping(modules_system): found_modules = [ m[2] for m in util.find_modules('testmod', - toolchain_mapping={ + environ_mapping={ r'.*_ba.*': 'builtin', r'testmod_foo': 'foo' }) From 2eebb1cc0d340518c32b237ac13cf2697c236327 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Fri, 18 Sep 2020 22:38:34 +0200 Subject: [PATCH 15/15] Remove stale comment --- unittests/test_utility.py | 1 - 1 file changed, 1 deletion(-) diff --git a/unittests/test_utility.py b/unittests/test_utility.py index ff90b54f4d..5199def6af 100644 --- a/unittests/test_utility.py +++ b/unittests/test_utility.py @@ -1326,7 +1326,6 @@ def user_exec_ctx(request, temp_runtime): @pytest.fixture def modules_system(user_exec_ctx, monkeypatch): # Pretend to be on a clean modules environment - # monkeypatch.setenv('MODULEPATH', '') monkeypatch.setenv('LOADEDMODULES', '') monkeypatch.setenv('_LMFILES_', '')