From cd46fb37c644c9c29b5fac2a87315680f7103815 Mon Sep 17 00:00:00 2001 From: Victor Holanda Date: Wed, 11 Sep 2019 13:27:34 +0200 Subject: [PATCH 1/6] Add support for modules 3.1 --- reframe/core/modules.py | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/reframe/core/modules.py b/reframe/core/modules.py index dea5c9af12..f2ea83a179 100644 --- a/reframe/core/modules.py +++ b/reframe/core/modules.py @@ -380,20 +380,32 @@ def __str__(self): class TModImpl(ModulesSystemImpl): """Module system for TMod (Tcl).""" - MIN_VERSION = (3, 2) + MIN_VERSION = (3, 1) def __init__(self): # Try to figure out if we are indeed using the TCL version + version_smaller_than_3_2 = False try: completed = os_ext.run_command('modulecmd -V') except OSError as e: - raise ConfigError( - 'could not find a sane TMod installation: %s' % e) from e - - version_match = re.search(r'^VERSION=(\S+)', completed.stdout, - re.MULTILINE) - tcl_version_match = re.search(r'^TCL_VERSION=(\S+)', completed.stdout, + try: + modulecmd = os.getenv('MODULESHOME') + modulecmd = os.path.join(modulecmd, 'modulecmd.tcl') + completed = os_ext.run_command(modulecmd) + version_smaller_than_3_2 = True + except OSError as e: + raise ConfigError( + 'could not find a sane TMod installation: %s' % e) from e + + if version_smaller_than_3_2: + version_match = re.search(r'Release Tcl (\S+)', completed.stderr, re.MULTILINE) + tcl_version_match = version_match + else: + version_match = re.search(r'^VERSION=(\S+)', completed.stdout, + re.MULTILINE) + tcl_version_match = re.search(r'^TCL_VERSION=(\S+)', completed.stdout, + re.MULTILINE) if version_match is None or tcl_version_match is None: raise ConfigError('could not find a sane TMod installation') @@ -412,6 +424,7 @@ def __init__(self): self._version = version self._command = 'modulecmd python' + try: # Try the Python bindings now completed = os_ext.run_command(self._command) From 56c32c1351d5bb9a91690d4454d5499cf54eb216 Mon Sep 17 00:00:00 2001 From: Victor Holanda Date: Wed, 23 Oct 2019 10:49:39 +0200 Subject: [PATCH 2/6] Add configuration specific to TMod version 3.1 Address PR remark to split TMod into two classes Add 'tmod31' as possible supported module --- reframe/core/modules.py | 187 +++++++++++++++++++++++++++------------- 1 file changed, 129 insertions(+), 58 deletions(-) diff --git a/reframe/core/modules.py b/reframe/core/modules.py index d022729ca7..2a3a3dd8bf 100644 --- a/reframe/core/modules.py +++ b/reframe/core/modules.py @@ -86,6 +86,8 @@ class ModulesSystem: def create(cls, modules_kind=None): if modules_kind is None: return ModulesSystem(NoModImpl()) + elif modules_kind == 'tmod31': + return ModulesSystem(TMod31Impl()) elif modules_kind == 'tmod': return ModulesSystem(TModImpl()) elif modules_kind == 'tmod4': @@ -377,64 +379,12 @@ def __str__(self): return self.name() + ' ' + self.version() -class TModImpl(ModulesSystemImpl): - '''Module system for TMod (Tcl).''' - - MIN_VERSION = (3, 1) +class TModBaseImpl(ModulesSystemImpl): + '''Base class for TMod Module system (Tcl).''' def __init__(self): - # Try to figure out if we are indeed using the TCL version - version_smaller_than_3_2 = False - try: - completed = os_ext.run_command('modulecmd -V') - except OSError as e: - try: - modulecmd = os.getenv('MODULESHOME') - modulecmd = os.path.join(modulecmd, 'modulecmd.tcl') - completed = os_ext.run_command(modulecmd) - version_smaller_than_3_2 = True - except OSError as e: - raise ConfigError( - 'could not find a sane TMod installation: %s' % e) from e - - if version_smaller_than_3_2: - version_match = re.search(r'Release Tcl (\S+)', completed.stderr, - re.MULTILINE) - tcl_version_match = version_match - else: - version_match = re.search(r'^VERSION=(\S+)', completed.stdout, - re.MULTILINE) - tcl_version_match = re.search(r'^TCL_VERSION=(\S+)', completed.stdout, - re.MULTILINE) - - if version_match is None or tcl_version_match is None: - raise ConfigError('could not find a sane TMod installation') - - version = version_match.group(1) - try: - ver_major, ver_minor, *_ = [int(v) for v in version.split('.')] - except ValueError: - raise ConfigError( - 'could not parse TMod version string: ' + version) from None - - if (ver_major, ver_minor) < self.MIN_VERSION: - raise ConfigError( - 'unsupported TMod version: %s (required >= %s)' % - (version, self.MIN_VERSION)) - - self._version = version - self._command = 'modulecmd python' - - try: - # Try the Python bindings now - completed = os_ext.run_command(self._command) - except OSError as e: - raise ConfigError( - 'could not get the Python bindings for TMod: ' % e) from e - - if re.search(r'Unknown shell type', completed.stderr): - raise ConfigError( - 'Python is not supported by this TMod installation') + self._version = None + self._command = None def name(self): return 'tmod' @@ -462,8 +412,7 @@ def _module_command_failed(self, completed): return re.search(r'ERROR', completed.stderr) is not None def _exec_module_command(self, *args, msg=None): - completed = self._run_module_command(*args, msg=msg) - exec(completed.stdout) + pass def loaded_modules(self): try: @@ -512,6 +461,128 @@ def emit_unload_instr(self, module): return 'module unload %s' % module +class TMod31Impl(TModBaseImpl): + '''Module system for TMod (Tcl).''' + + MIN_VERSION = (3, 1) + + def __init__(self): + # Try to figure out if we are indeed using the TCL version + try: + modulecmd = os.getenv('MODULESHOME') + modulecmd = os.path.join(modulecmd, 'modulecmd.tcl') + completed = os_ext.run_command(modulecmd) + except OSError as e: + raise ConfigError( + 'could not find a sane TMod31 installation: %s' % e) from e + + version_match = re.search(r'Release Tcl (\S+)', completed.stderr, + re.MULTILINE) + tcl_version_match = version_match + + if version_match is None or tcl_version_match is None: + raise ConfigError('could not find a sane TMod31 installation') + + version = version_match.group(1) + try: + ver_major, ver_minor, *_ = [int(v) for v in version.split('.')] + except ValueError: + raise ConfigError( + 'could not parse TMod31 version string: ' + version) from None + + if (ver_major, ver_minor) < self.MIN_VERSION: + raise ConfigError( + 'unsupported TMod version: %s (required >= %s)' % + (version, self.MIN_VERSION)) + + self._version = version + self._command = '%s python' % modulecmd + + try: + # Try the Python bindings now + completed = os_ext.run_command(self._command) + except OSError as e: + raise ConfigError( + 'could not get the Python bindings for TMod31: ' % e) from e + + if re.search(r'Unknown shell type', completed.stderr): + raise ConfigError( + 'Python is not supported by this TMod installation') + + def name(self): + return 'tmod31' + + def _exec_module_command(self, *args, msg=None): + completed = self._run_module_command(*args, msg=msg) + exec_match = re.search(r'^exec\s\'', completed.stdout) + if exec_match is None: + raise ConfigError('could not use the python bindings') + else: + cmd = completed.stdout + exec_match = re.search(r'^exec\s\'(\S+)\'', cmd, + re.MULTILINE) + if exec_match is None: + raise ConfigError('could not use the python bindings') + with open(exec_match.group(1), 'r') as content_file: + cmd = content_file.read() + + exec(cmd) + + +class TModImpl(TModBaseImpl): + '''Module system for TMod (Tcl).''' + + MIN_VERSION = (3, 2) + + def __init__(self): + # Try to figure out if we are indeed using the TCL version + try: + completed = os_ext.run_command('modulecmd -V') + except OSError as e: + raise ConfigError( + 'could not find a sane TMod installation: %s' % e) from e + + version_match = re.search(r'^VERSION=(\S+)', completed.stdout, + re.MULTILINE) + tcl_version_match = re.search(r'^TCL_VERSION=(\S+)', completed.stdout, + re.MULTILINE) + + if version_match is None or tcl_version_match is None: + raise ConfigError('could not find a sane TMod installation') + + version = version_match.group(1) + try: + ver_major, ver_minor, *_ = [int(v) for v in version.split('.')] + except ValueError: + raise ConfigError( + 'could not parse TMod version string: ' + version) from None + + if (ver_major, ver_minor) < self.MIN_VERSION: + raise ConfigError( + 'unsupported TMod version: %s (required >= %s)' % + (version, self.MIN_VERSION)) + + self._version = version + self._command = 'modulecmd python' + try: + # Try the Python bindings now + completed = os_ext.run_command(self._command) + except OSError as e: + raise ConfigError( + 'could not get the Python bindings for TMod: ' % e) from e + + if re.search(r'Unknown shell type', completed.stderr): + raise ConfigError( + 'Python is not supported by this TMod installation') + + def name(self): + return 'tmod' + + def _exec_module_command(self, *args, msg=None): + completed = self._run_module_command(*args, msg=msg) + exec(completed.stdout) + + class TMod4Impl(TModImpl): '''Module system for TMod 4.''' From 509b15f35a6d60b2253bf49e6cda0fe6e55c339c Mon Sep 17 00:00:00 2001 From: Victor Holanda Date: Fri, 15 Nov 2019 09:38:54 +0100 Subject: [PATCH 3/6] Address PR remarks --- reframe/core/modules.py | 107 ++++++++++++++++++---------------------- 1 file changed, 48 insertions(+), 59 deletions(-) diff --git a/reframe/core/modules.py b/reframe/core/modules.py index 2a3a3dd8bf..24083abe10 100644 --- a/reframe/core/modules.py +++ b/reframe/core/modules.py @@ -90,6 +90,8 @@ def create(cls, modules_kind=None): return ModulesSystem(TMod31Impl()) elif modules_kind == 'tmod': return ModulesSystem(TModImpl()) + elif modules_kind == 'tmod32': + return ModulesSystem(TModImpl()) elif modules_kind == 'tmod4': return ModulesSystem(TMod4Impl()) elif modules_kind == 'lmod': @@ -379,12 +381,52 @@ def __str__(self): return self.name() + ' ' + self.version() -class TModBaseImpl(ModulesSystemImpl): +class TModImpl(ModulesSystemImpl): '''Base class for TMod Module system (Tcl).''' + MIN_VERSION = (3, 2) + def __init__(self): - self._version = None - self._command = None + # Try to figure out if we are indeed using the TCL version + try: + completed = os_ext.run_command('modulecmd -V') + except OSError as e: + raise ConfigError( + 'could not find a sane TMod installation: %s' % e) from e + + version_match = re.search(r'^VERSION=(\S+)', completed.stdout, + re.MULTILINE) + tcl_version_match = re.search(r'^TCL_VERSION=(\S+)', completed.stdout, + re.MULTILINE) + + if version_match is None or tcl_version_match is None: + raise ConfigError('could not find a sane TMod installation') + + version = version_match.group(1) + try: + ver_major, ver_minor, *_ = [int(v) for v in version.split('.')] + except ValueError: + raise ConfigError( + 'could not parse TMod version string: ' + version) from None + + if (ver_major, ver_minor) < self.MIN_VERSION: + raise ConfigError( + 'unsupported TMod version: %s (required >= %s)' % + (version, self.MIN_VERSION)) + + self._version = version + self._command = 'modulecmd python' + try: + # Try the Python bindings now + completed = os_ext.run_command(self._command) + except OSError as e: + raise ConfigError( + 'could not get the Python bindings for TMod: ' % e) from e + + if re.search(r'Unknown shell type', completed.stderr): + raise ConfigError( + 'Python is not supported by this TMod installation') + def name(self): return 'tmod' @@ -412,7 +454,8 @@ def _module_command_failed(self, completed): return re.search(r'ERROR', completed.stderr) is not None def _exec_module_command(self, *args, msg=None): - pass + completed = self._run_module_command(*args, msg=msg) + exec(completed.stdout) def loaded_modules(self): try: @@ -461,7 +504,7 @@ def emit_unload_instr(self, module): return 'module unload %s' % module -class TMod31Impl(TModBaseImpl): +class TMod31Impl(ModulesSystemImpl): '''Module system for TMod (Tcl).''' MIN_VERSION = (3, 1) @@ -529,60 +572,6 @@ def _exec_module_command(self, *args, msg=None): exec(cmd) -class TModImpl(TModBaseImpl): - '''Module system for TMod (Tcl).''' - - MIN_VERSION = (3, 2) - - def __init__(self): - # Try to figure out if we are indeed using the TCL version - try: - completed = os_ext.run_command('modulecmd -V') - except OSError as e: - raise ConfigError( - 'could not find a sane TMod installation: %s' % e) from e - - version_match = re.search(r'^VERSION=(\S+)', completed.stdout, - re.MULTILINE) - tcl_version_match = re.search(r'^TCL_VERSION=(\S+)', completed.stdout, - re.MULTILINE) - - if version_match is None or tcl_version_match is None: - raise ConfigError('could not find a sane TMod installation') - - version = version_match.group(1) - try: - ver_major, ver_minor, *_ = [int(v) for v in version.split('.')] - except ValueError: - raise ConfigError( - 'could not parse TMod version string: ' + version) from None - - if (ver_major, ver_minor) < self.MIN_VERSION: - raise ConfigError( - 'unsupported TMod version: %s (required >= %s)' % - (version, self.MIN_VERSION)) - - self._version = version - self._command = 'modulecmd python' - try: - # Try the Python bindings now - completed = os_ext.run_command(self._command) - except OSError as e: - raise ConfigError( - 'could not get the Python bindings for TMod: ' % e) from e - - if re.search(r'Unknown shell type', completed.stderr): - raise ConfigError( - 'Python is not supported by this TMod installation') - - def name(self): - return 'tmod' - - def _exec_module_command(self, *args, msg=None): - completed = self._run_module_command(*args, msg=msg) - exec(completed.stdout) - - class TMod4Impl(TModImpl): '''Module system for TMod 4.''' From 98e062f06a436c9d882dbf06a0c9b69d91abbe19 Mon Sep 17 00:00:00 2001 From: Victor Holanda Rusu Date: Sat, 16 Nov 2019 13:01:10 +0100 Subject: [PATCH 4/6] Fix TMod 3.1 hierarchy --- reframe/core/modules.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/reframe/core/modules.py b/reframe/core/modules.py index 24083abe10..c8efba0c72 100644 --- a/reframe/core/modules.py +++ b/reframe/core/modules.py @@ -384,7 +384,7 @@ def __str__(self): class TModImpl(ModulesSystemImpl): '''Base class for TMod Module system (Tcl).''' - MIN_VERSION = (3, 2) + MIN_VERSION = (3, 2) def __init__(self): # Try to figure out if we are indeed using the TCL version @@ -504,7 +504,7 @@ def emit_unload_instr(self, module): return 'module unload %s' % module -class TMod31Impl(ModulesSystemImpl): +class TMod31Impl(TModImpl): '''Module system for TMod (Tcl).''' MIN_VERSION = (3, 1) From e28b3d0b2279af96573155ade0bf7c157c257eb6 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Mon, 25 Nov 2019 22:24:36 +0100 Subject: [PATCH 5/6] Add documentation for the Tmod 3.1 backend --- docs/configure.rst | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/docs/configure.rst b/docs/configure.rst index 3ea07e667f..7c91346022 100644 --- a/docs/configure.rst +++ b/docs/configure.rst @@ -104,17 +104,18 @@ The valid attributes of a system are the following: * ``descr``: A detailed description of the system (default is the system name). * ``hostnames``: This is a list of hostname patterns that will be used by ReFrame when it tries to `auto-detect <#system-auto-detection>`__ the current system (default ``[]``). -* ``modules_system``: The modules system that should be used for loading environment modules on this system (default :class:`None`). +* ``modules_system``: *[new in 2.8]* The modules system that should be used for loading environment modules on this system (default :class:`None`). Three types of modules systems are currently supported: - - ``tmod``: The classic Tcl implementation of the `environment modules `__ (versions older than 3.2 are not supported). + - ``tmod`` or ``tmod32``: The classic Tcl implementation of the `environment modules `__ (version 3.2). + - ``tmod31``: *[new in 2.21]* The classic Tcl implementation of the `environment modules `__ (version 3.1). - ``tmod4``: The version 4 of the Tcl implementation of the `environment modules `__ (versions older than 4.1 are not supported). - ``lmod``: The Lua implementation of the `environment modules `__. -* ``modules``: Modules to be loaded always when running on this system. +* ``modules``: *[new in 2.19]* Modules to be loaded always when running on this system. These modules modify the ReFrame environment. This is useful when for example a particular module is needed to submit jobs on a specific system. -* ``variables``: Environment variables to be set always when running on this system. +* ``variables``: *[new in 2.19]* Environment variables to be set always when running on this system. * ``prefix``: Default regression prefix for this system (default ``.``). * ``stagedir``: Default stage directory for this system (default :class:`None`). * ``outputdir``: Default output directory for this system (default :class:`None`). @@ -126,13 +127,7 @@ The valid attributes of a system are the following: For a more detailed description of the ``prefix``, ``stagedir``, ``outputdir`` and ``perflogdir`` directories, please refer to the `"Configuring ReFrame Directories" `__ and `"Performance Logging" `__ sections. .. note:: - .. versionadded:: 2.8 - The ``modules_system`` key was introduced for specifying custom modules systems for different systems. - -.. note:: - .. versionadded:: 2.19 - The ``modules`` and ``variables`` configuration parameters were introduced at the system level. - + A different backend is used for Tmod 3.1, due to its different Python bindings. .. warning:: .. versionchanged:: 2.18 From 96e1153c0bacfa8052e9c6dc049153e69a0e708f Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Mon, 25 Nov 2019 22:26:47 +0100 Subject: [PATCH 6/6] Fix PEP8 issue --- reframe/core/modules.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/reframe/core/modules.py b/reframe/core/modules.py index c8efba0c72..c66488ea22 100644 --- a/reframe/core/modules.py +++ b/reframe/core/modules.py @@ -427,7 +427,6 @@ def __init__(self): raise ConfigError( 'Python is not supported by this TMod installation') - def name(self): return 'tmod' @@ -563,7 +562,7 @@ def _exec_module_command(self, *args, msg=None): else: cmd = completed.stdout exec_match = re.search(r'^exec\s\'(\S+)\'', cmd, - re.MULTILINE) + re.MULTILINE) if exec_match is None: raise ConfigError('could not use the python bindings') with open(exec_match.group(1), 'r') as content_file: