From 8a62ce9bc0b40f980ad8ac63450c5ed2c48e227d Mon Sep 17 00:00:00 2001 From: rafael Date: Tue, 22 Feb 2022 17:18:27 +0100 Subject: [PATCH 01/12] specify hostname cmds through config --- config/cscs.py | 1 + reframe/core/config.py | 17 +++++++---------- reframe/frontend/cli.py | 5 ++++- reframe/schemas/config.json | 9 +++++++-- reframe/utility/__init__.py | 18 ++++++++++++++++++ 5 files changed, 37 insertions(+), 13 deletions(-) diff --git a/config/cscs.py b/config/cscs.py index 72c765ab4a..7bb73d2146 100644 --- a/config/cscs.py +++ b/config/cscs.py @@ -1008,6 +1008,7 @@ { 'check_search_path': ['checks/'], 'check_search_recursive': True, + 'hostname_cmd': 'xthostname', 'remote_detect': True } ] diff --git a/reframe/core/config.py b/reframe/core/config.py index fda75aeaee..a3a9d0f952 100644 --- a/reframe/core/config.py +++ b/reframe/core/config.py @@ -20,7 +20,7 @@ from reframe.core.environments import normalize_module_list from reframe.core.exceptions import ConfigError, ReframeFatalError from reframe.core.logging import getlogger -from reframe.utility import ScopedDict +from reframe.utility import ScopedDict, get_hostname_cmd def _match_option(opt, opt_map): @@ -261,16 +261,13 @@ def _create_from_json(cls, filename): def _detect_system(self): getlogger().debug('Detecting system') - if os.path.exists('/etc/xthostname'): - # Get the cluster name on Cray systems - getlogger().debug( - "Found '/etc/xthostname': will use this to get the system name" - ) - with open('/etc/xthostname') as fp: - hostname = fp.read() - else: - hostname = socket.getfqdn() + try: + hostname_cmd = self._site_config['general'][0].get('hostname_cmd', + 'hostname') + except (KeyError, IndexError): + hostname_cmd = 'hostname' + hostname = get_hostname_cmd(hostname_cmd, getlogger()) getlogger().debug( f'Looking for a matching configuration entry ' f'for system {hostname!r}' diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index 026473278a..9a95c18905 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -34,6 +34,7 @@ from reframe.frontend.executors.policies import (SerialExecutionPolicy, AsynchronousExecutionPolicy) from reframe.frontend.executors import Runner, generate_testcases +from reframe.utility import get_hostname_cmd def format_env(envvars): @@ -877,7 +878,9 @@ def print_infoline(param, value): 'cmdline': ' '.join(sys.argv), 'config_file': rt.site_config.filename, 'data_version': runreport.DATA_VERSION, - 'hostname': socket.getfqdn(), + 'hostname': get_hostname_cmd( + site_config.get('general/0/hostname_cmd') + ), 'prefix_output': rt.output_prefix, 'prefix_stage': rt.stage_prefix, 'user': osext.osuser(), diff --git a/reframe/schemas/config.json b/reframe/schemas/config.json index 634028346c..83e360f07c 100644 --- a/reframe/schemas/config.json +++ b/reframe/schemas/config.json @@ -456,8 +456,11 @@ "colorize": {"type": "boolean"}, "compact_test_names": {"type": "boolean"}, "git_timeout": {"type": "number"}, + "hostname_cmd": { + "type": "string", + "enum": ["fqdn", "hostname", "xthostname"] + }, "ignore_check_conflicts": {"type": "boolean"}, - "trap_job_errors": {"type": "boolean"}, "keep_stage_files": {"type": "boolean"}, "module_map_file": {"type": "string"}, "module_mappings": { @@ -476,6 +479,7 @@ "save_log_files": {"type": "boolean"}, "target_systems": {"$ref": "#/defs/system_ref"}, "timestamp_dirs": {"type": "string"}, + "trap_job_errors": {"type": "boolean"}, "unload_modules": {"$ref": "#/defs/modules_list"}, "use_login_shell": {"type": "boolean"}, "user_modules": {"$ref": "#/defs/modules_list"}, @@ -508,8 +512,8 @@ "general/colorize": true, "general/compact_test_names": false, "general/git_timeout": 5, + "general/hostname_cmd": "hostname", "general/ignore_check_conflicts": false, - "general/trap_job_errors": false, "general/keep_stage_files": false, "general/module_map_file": "", "general/module_mappings": [], @@ -523,6 +527,7 @@ "general/save_log_files": false, "general/target_systems": ["*"], "general/timestamp_dirs": "", + "general/trap_job_errors": false, "general/unload_modules": [], "general/use_login_shell": false, "general/user_modules": [], diff --git a/reframe/utility/__init__.py b/reframe/utility/__init__.py index 2fa3255ac1..62d75bb5ff 100644 --- a/reframe/utility/__init__.py +++ b/reframe/utility/__init__.py @@ -13,6 +13,7 @@ import itertools import os import re +import socket import sys import types import weakref @@ -24,6 +25,23 @@ from . import typecheck as typ +def get_hostname_cmd(hostname_cmd, logger=None): + if hostname_cmd == 'xthostname': + if os.path.exists('/etc/xthostnamex'): + # Get the cluster name on Cray systems + with open('/etc/xthostname') as fp: + return fp.read() + elif logger: + logger.debug("Did not find '/etc/xthostname': will use " + "'socket.gethostname' to get the system name") + return socket.gethostname() + + elif hostname_cmd == 'fqdn': + return socket.getfqdn() + else: + return socket.gethostname() + + def seconds_to_hms(seconds): '''Convert time in seconds to hours, minutes and seconds. From f875d375d4868a90c68d5e0481e3e0f824bb0e65 Mon Sep 17 00:00:00 2001 From: rafael Date: Tue, 22 Feb 2022 18:03:30 +0100 Subject: [PATCH 02/12] remove unused import --- reframe/core/config.py | 1 - 1 file changed, 1 deletion(-) diff --git a/reframe/core/config.py b/reframe/core/config.py index a3a9d0f952..6f5033f9d6 100644 --- a/reframe/core/config.py +++ b/reframe/core/config.py @@ -11,7 +11,6 @@ import jsonschema import os import re -import socket import tempfile import reframe From 9446ee2e8b5d7fda7244f811d1c9581196765df5 Mon Sep 17 00:00:00 2001 From: rafael Date: Tue, 22 Feb 2022 18:49:55 +0100 Subject: [PATCH 03/12] remove unused import --- reframe/frontend/cli.py | 1 - 1 file changed, 1 deletion(-) diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index 9a95c18905..d6bd3d1855 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -8,7 +8,6 @@ import json import os import shlex -import socket import sys import time import traceback From 2a64594450c53194361e1944a7865227600880a6 Mon Sep 17 00:00:00 2001 From: Rafael Sarmiento Date: Wed, 23 Feb 2022 20:27:59 +0100 Subject: [PATCH 04/12] fix typo and add docs --- docs/config_reference.rst | 13 +++++++++++++ docs/configure.rst | 11 ++++++++--- docs/manpage.rst | 1 - reframe/utility/__init__.py | 2 +- 4 files changed, 22 insertions(+), 5 deletions(-) diff --git a/docs/config_reference.rst b/docs/config_reference.rst index 12b64dcde6..b3d5f72b61 100644 --- a/docs/config_reference.rst +++ b/docs/config_reference.rst @@ -1315,6 +1315,19 @@ General Configuration .. versionadded:: 3.9.0 +.. js:attribute:: .general[].hostname_cmd + + :required: No + :default: ``'hostname'`` + + Command to determine the systems name. + There are three values available: ``'hostname'``, ``fqdn`` and ``'xthostname'``. + The values ``'hostname'`` and ``'fqdn'`` can be specified to detect the system using Python's ``socket.gethostname()`` or ``socket.getfqdn()`` respectively. + With the ``'xthostname'`` option, ReFrame first tries to obtain the hostname from ``/etc/xthostname``, which provides the unqualified machine name in Cray systems. + If this cannot be found, ReFrame falls back to the ``'hostname'`` option. + + .. versionadded:: 3.11.0 + .. js:attribute:: .general[].git_timeout :required: No diff --git a/docs/configure.rst b/docs/configure.rst index 58b041cf37..b57b8d8e55 100644 --- a/docs/configure.rst +++ b/docs/configure.rst @@ -235,9 +235,14 @@ Picking a System Configuration As discussed previously, ReFrame's configuration file can store the configurations for multiple systems. When launched, ReFrame will pick the first matching configuration and load it. -This process is performed as follows: -ReFrame first tries to obtain the hostname from ``/etc/xthostname``, which provides the unqualified *machine name* in Cray systems. -If this cannot be found, the hostname will be obtained from the standard ``hostname`` command. + +There are three ways in which the system name can be determined. +They can be specified in the configuration through the ``hostname_cmd`` option which can take one of the three values ``'hostname'``, ``fqdn`` or ``'xthostname'``. +When ``hostname_cmd`` is set to ``'hostname'``, which is the default value, ReFrame will detect the system using Python's ``socket.gethostname()``. +Similarly, with ``'hostname'``, ReFrame will use ``socket.getfqdn()``. +With ``'xthostname'``, ReFrame first tries to obtain the hostname from ``/etc/xthostname``, which provides the unqualified *machine name* in Cray systems. +If this cannot be found, ReFrame falls back to the ``'hostname'`` option. + Having retrieved the hostname, ReFrame goes through all the systems in its configuration and tries to match the hostname against any of the patterns defined in each system's ``hostnames`` property. The detection process stops at the first match found, and that system's configuration is selected. diff --git a/docs/manpage.rst b/docs/manpage.rst index 114587641e..392051f76d 100644 --- a/docs/manpage.rst +++ b/docs/manpage.rst @@ -784,7 +784,6 @@ Miscellaneous options If this option is not specified, ReFrame will try to pick the correct configuration entry automatically. It does so by trying to match the hostname of the current machine again the hostname patterns defined in the :js:attr:`hostnames` system configuration parameter. The system with the first match becomes the current system. - For Cray systems, ReFrame will first look for the *unqualified machine name* in ``/etc/xthostname`` before trying retrieving the hostname of the current machine. This option can also be set using the :envvar:`RFM_SYSTEM` environment variable. diff --git a/reframe/utility/__init__.py b/reframe/utility/__init__.py index 62d75bb5ff..32046218b0 100644 --- a/reframe/utility/__init__.py +++ b/reframe/utility/__init__.py @@ -27,7 +27,7 @@ def get_hostname_cmd(hostname_cmd, logger=None): if hostname_cmd == 'xthostname': - if os.path.exists('/etc/xthostnamex'): + if os.path.exists('/etc/xthostname'): # Get the cluster name on Cray systems with open('/etc/xthostname') as fp: return fp.read() From ebee3c3b50907c84df1cbbe075224eb6c51aad3d Mon Sep 17 00:00:00 2001 From: Rafael Sarmiento Date: Thu, 10 Mar 2022 16:38:19 +0100 Subject: [PATCH 05/12] reimplement with env vars --- config/cscs.py | 1 - docs/config_reference.rst | 13 -------- docs/configure.rst | 12 ++++---- docs/manpage.rst | 52 ++++++++++++++++++++++++++++++++ reframe/core/config.py | 29 ++++++++++-------- reframe/frontend/cli.py | 60 +++++++++++++++++++++++++++++-------- reframe/utility/__init__.py | 23 +++++++++----- 7 files changed, 138 insertions(+), 52 deletions(-) diff --git a/config/cscs.py b/config/cscs.py index 7bb73d2146..72c765ab4a 100644 --- a/config/cscs.py +++ b/config/cscs.py @@ -1008,7 +1008,6 @@ { 'check_search_path': ['checks/'], 'check_search_recursive': True, - 'hostname_cmd': 'xthostname', 'remote_detect': True } ] diff --git a/docs/config_reference.rst b/docs/config_reference.rst index fe1fe53296..466b079651 100644 --- a/docs/config_reference.rst +++ b/docs/config_reference.rst @@ -1316,19 +1316,6 @@ General Configuration .. versionadded:: 3.9.0 -.. js:attribute:: .general[].hostname_cmd - - :required: No - :default: ``'hostname'`` - - Command to determine the systems name. - There are three values available: ``'hostname'``, ``fqdn`` and ``'xthostname'``. - The values ``'hostname'`` and ``'fqdn'`` can be specified to detect the system using Python's ``socket.gethostname()`` or ``socket.getfqdn()`` respectively. - With the ``'xthostname'`` option, ReFrame first tries to obtain the hostname from ``/etc/xthostname``, which provides the unqualified machine name in Cray systems. - If this cannot be found, ReFrame falls back to the ``'hostname'`` option. - - .. versionadded:: 3.11.0 - .. js:attribute:: .general[].git_timeout :required: No diff --git a/docs/configure.rst b/docs/configure.rst index b57b8d8e55..fe7221dc72 100644 --- a/docs/configure.rst +++ b/docs/configure.rst @@ -236,12 +236,12 @@ Picking a System Configuration As discussed previously, ReFrame's configuration file can store the configurations for multiple systems. When launched, ReFrame will pick the first matching configuration and load it. -There are three ways in which the system name can be determined. -They can be specified in the configuration through the ``hostname_cmd`` option which can take one of the three values ``'hostname'``, ``fqdn`` or ``'xthostname'``. -When ``hostname_cmd`` is set to ``'hostname'``, which is the default value, ReFrame will detect the system using Python's ``socket.gethostname()``. -Similarly, with ``'hostname'``, ReFrame will use ``socket.getfqdn()``. -With ``'xthostname'``, ReFrame first tries to obtain the hostname from ``/etc/xthostname``, which provides the unqualified *machine name* in Cray systems. -If this cannot be found, ReFrame falls back to the ``'hostname'`` option. +The way in which the system name is determined can be controlled by the environment variables ``RFM_AUTODETECT_METHOD``, ``RFM_AUTODETECT_FQDN`` and ``RFM_AUTODETECT_XTHOSTNAME``. +``RFM_AUTODETECT_METHOD`` is a string that specifies the method used to detect the system name. +Its default value is ``'hostname'``, which currently is the only supported option. +The host name will be determined by ``socket.gethostname()``, unless the boolean ``RFM_AUTODETECT_FQDN`` is set, which make ReFrame use ``socket.getfqdn()`` instead. +The last option, ``RFM_AUTODETECT_XTHOSTNAME`` is a boolean and when set, ReFrame first tries to obtain the hostname from ``/etc/xthostname``, which provides the unqualified *machine name* in Cray systems. +If this cannot be found, ReFrame falls back to the one of the two options set by ``RFM_AUTODETECT_FQDN``. Having retrieved the hostname, ReFrame goes through all the systems in its configuration and tries to match the hostname against any of the patterns defined in each system's ``hostnames`` property. The detection process stops at the first match found, and that system's configuration is selected. diff --git a/docs/manpage.rst b/docs/manpage.rst index 392051f76d..3c66891344 100644 --- a/docs/manpage.rst +++ b/docs/manpage.rst @@ -1022,6 +1022,58 @@ Here is an alphabetical list of the environment variables recognized by ReFrame: ================================== ================== +.. envvar:: RFM_AUTODETECT_METHOD + + Method to detect the system. + + There is only one method defined + + - ``hostname``: The system will be detected by finding its hostname. + + .. table:: + :align: left + + ================================== ================== + Associated command line option N/A + Associated configuration parameter N/A + ================================== ================== + + + .. versionadded:: 3.12.0 + + +.. envvar:: RFM_AUTODETECT_FQDN + + Use FQDN as host name. + + .. table:: + :align: left + + ================================== ================== + Associated command line option N/A + Associated configuration parameter N/A + ================================== ================== + + + .. versionadded:: 3.12.0 + + +.. envvar:: RFM_AUTODETECT_XTHOSTNAME + + Use Cray's xthostname file to find the host name. + + .. table:: + :align: left + + ================================== ================== + Associated command line option N/A + Associated configuration parameter N/A + ================================== ================== + + + .. versionadded:: 3.12.0 + + .. envvar:: RFM_GIT_TIMEOUT Timeout value in seconds used when checking if a git repository exists. diff --git a/reframe/core/config.py b/reframe/core/config.py index 459ecd0639..1289aec516 100644 --- a/reframe/core/config.py +++ b/reframe/core/config.py @@ -213,15 +213,19 @@ def subconfig_system(self): return self._local_system @classmethod - def create(cls, filename): + def create(cls, filename, autodetect_opts): _, ext = os.path.splitext(filename) if ext == '.py': - return cls._create_from_python(filename) + ret = cls._create_from_python(filename) elif ext == '.json': - return cls._create_from_json(filename) + ret = cls._create_from_json(filename) else: raise ConfigError(f"unknown configuration file type: '{filename}'") + ret.autodetect_opts = autodetect_opts + + return ret + @classmethod def _create_from_python(cls, filename): try: @@ -263,13 +267,7 @@ def _create_from_json(cls, filename): def _detect_system(self): getlogger().debug('Detecting system') - try: - hostname_cmd = self._site_config['general'][0].get('hostname_cmd', - 'hostname') - except (KeyError, IndexError): - hostname_cmd = 'hostname' - - hostname = get_hostname_cmd(hostname_cmd, getlogger()) + hostname = get_hostname_cmd(self.autodetect_opts, getlogger()) getlogger().debug( f'Looking for a matching configuration entry ' f'for system {hostname!r}' @@ -621,7 +619,14 @@ def _find_config_file(): return None -def load_config(filename=None): +def load_config(filename=None, autodetect_opts=None): + if not autodetect_opts: + autodetect_opts = { + 'autodetect_method': 'hostname', + 'autodetect_xthostname': True, + 'autodetect_fqdn': True, + } + if filename is None: filename = _find_config_file() if filename is None: @@ -631,4 +636,4 @@ def load_config(filename=None): return _SiteConfig(settings.site_configuration, '') getlogger().debug(f'Loading configuration file: {filename!r}') - return _SiteConfig.create(filename) + return _SiteConfig.create(filename, autodetect_opts) diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index d6bd3d1855..d0b97ec59e 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -523,6 +523,32 @@ def main(): ) # Options not associated with command-line arguments + argparser.add_argument( + dest='use_login_shell', + envvar='RFM_USE_LOGIN_SHELL', + configvar='general/use_login_shell', + action='store_true', + help='Use a login shell for job scripts' + ) + argparser.add_argument( + dest='autodetect_method', + envvar='RFM_AUTODETECT_METHOD', + action='store', + help='Method to detect the system' + ) + argparser.add_argument( + dest='autodetect_fqdn', + envvar='RFM_AUTODETECT_FQDN', + action='store_true', + help='Use FQDN as host name' + ) + argparser.add_argument( + dest='autodetect_xthostname', + envvar='RFM_AUTODETECT_XTHOSTNAME', + action='store_false', + default=True, + help="Use Cray's xthostname file to find the host name" + ) argparser.add_argument( dest='git_timeout', envvar='RFM_GIT_TIMEOUT', @@ -607,13 +633,6 @@ def main(): action='store_true', help='Trap job errors in job scripts and fail tests automatically' ) - argparser.add_argument( - dest='use_login_shell', - envvar='RFM_USE_LOGIN_SHELL', - configvar='general/use_login_shell', - action='store_true', - help='Use a login shell for job scripts' - ) def restrict_logging(): '''Restrict logging to errors only. @@ -640,11 +659,22 @@ def restrict_logging(): argparser.print_help() sys.exit(1) + autodetect_opts = { + 'autodetect_method': options.autodetect_method or 'hostname', + 'autodetect_xthostname': (options.autodetect_xthostname + if options.autodetect_xthostname is not None + else True), + 'autodetect_fqdn': (options.autodetect_fqdn + if options.autodetect_fqdn is not None + else True), + } + # First configure logging with our generic configuration so as to be able # to print pretty messages; logging will be reconfigured by user's # configuration later site_config = config.load_config( - os.path.join(reframe.INSTALL_PREFIX, 'reframe/core/settings.py') + os.path.join(reframe.INSTALL_PREFIX, 'reframe/core/settings.py'), + autodetect_opts=autodetect_opts ) site_config.select_subconfig('generic') options.update_config(site_config) @@ -679,11 +709,17 @@ def restrict_logging(): ) sys.exit(0) + if autodetect_opts['autodetect_method'] not in ['hostname']: + printer.error('unknown autodetect method ' + f"`{options.autodetect_method}': Exiting...") + sys.exit(1) + # Now configure ReFrame according to the user configuration file try: try: printer.debug('Loading user configuration') - site_config = config.load_config(options.config_file) + site_config = config.load_config(options.config_file, + autodetect_opts) except warnings.ReframeDeprecationWarning as e: printer.warning(e) converted = config.convert_old_config(options.config_file) @@ -691,7 +727,7 @@ def restrict_logging(): f"configuration file has been converted " f"to the new syntax here: '{converted}'" ) - site_config = config.load_config(converted) + site_config = config.load_config(converted, autodetect_opts) site_config.validate() @@ -877,9 +913,7 @@ def print_infoline(param, value): 'cmdline': ' '.join(sys.argv), 'config_file': rt.site_config.filename, 'data_version': runreport.DATA_VERSION, - 'hostname': get_hostname_cmd( - site_config.get('general/0/hostname_cmd') - ), + 'hostname': get_hostname_cmd(autodetect_opts), 'prefix_output': rt.output_prefix, 'prefix_stage': rt.stage_prefix, 'user': osext.osuser(), diff --git a/reframe/utility/__init__.py b/reframe/utility/__init__.py index 32046218b0..84e8d9aff0 100644 --- a/reframe/utility/__init__.py +++ b/reframe/utility/__init__.py @@ -25,18 +25,27 @@ from . import typecheck as typ -def get_hostname_cmd(hostname_cmd, logger=None): - if hostname_cmd == 'xthostname': +def get_hostname_cmd(autodetect_opts, logger=None): + if autodetect_opts['autodetect_xthostname']: if os.path.exists('/etc/xthostname'): # Get the cluster name on Cray systems with open('/etc/xthostname') as fp: return fp.read() - elif logger: - logger.debug("Did not find '/etc/xthostname': will use " - "'socket.gethostname' to get the system name") - return socket.gethostname() - elif hostname_cmd == 'fqdn': + elif autodetect_opts['autodetect_fqdn']: + if logger: + logger.debug("Did not find '/etc/xthostname': will use " + "'socket.getfqdn' to get the system name") + + return socket.getfqdn() + else: + if logger: + logger.debug("Did not find '/etc/xthostname': will use " + "'socket.gethostname' to get the system name") + + return socket.gethostname() + + elif autodetect_opts['autodetect_fqdn']: return socket.getfqdn() else: return socket.gethostname() From bf03e85d6a6cc38883bb1597e2e8634312afe272 Mon Sep 17 00:00:00 2001 From: Rafael Sarmiento Date: Thu, 10 Mar 2022 17:02:13 +0100 Subject: [PATCH 06/12] fix comments --- reframe/frontend/cli.py | 1 - reframe/schemas/config.json | 5 ----- reframe/utility/__init__.py | 21 +++++++++------------ 3 files changed, 9 insertions(+), 18 deletions(-) diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index d0b97ec59e..5174ede43b 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -674,7 +674,6 @@ def restrict_logging(): # configuration later site_config = config.load_config( os.path.join(reframe.INSTALL_PREFIX, 'reframe/core/settings.py'), - autodetect_opts=autodetect_opts ) site_config.select_subconfig('generic') options.update_config(site_config) diff --git a/reframe/schemas/config.json b/reframe/schemas/config.json index 83e360f07c..2f72158748 100644 --- a/reframe/schemas/config.json +++ b/reframe/schemas/config.json @@ -456,10 +456,6 @@ "colorize": {"type": "boolean"}, "compact_test_names": {"type": "boolean"}, "git_timeout": {"type": "number"}, - "hostname_cmd": { - "type": "string", - "enum": ["fqdn", "hostname", "xthostname"] - }, "ignore_check_conflicts": {"type": "boolean"}, "keep_stage_files": {"type": "boolean"}, "module_map_file": {"type": "string"}, @@ -512,7 +508,6 @@ "general/colorize": true, "general/compact_test_names": false, "general/git_timeout": 5, - "general/hostname_cmd": "hostname", "general/ignore_check_conflicts": false, "general/keep_stage_files": false, "general/module_map_file": "", diff --git a/reframe/utility/__init__.py b/reframe/utility/__init__.py index 84e8d9aff0..70da820528 100644 --- a/reframe/utility/__init__.py +++ b/reframe/utility/__init__.py @@ -32,22 +32,19 @@ def get_hostname_cmd(autodetect_opts, logger=None): with open('/etc/xthostname') as fp: return fp.read() - elif autodetect_opts['autodetect_fqdn']: - if logger: - logger.debug("Did not find '/etc/xthostname': will use " - "'socket.getfqdn' to get the system name") + elif logger: + logger.debug("Did not find '/etc/xthostname')") - return socket.getfqdn() - else: - if logger: - logger.debug("Did not find '/etc/xthostname': will use " - "'socket.gethostname' to get the system name") - - return socket.gethostname() + if autodetect_opts['autodetect_fqdn']: + if logger: + logger.debug("Will use 'socket.getfqdn' to get the system name") - elif autodetect_opts['autodetect_fqdn']: return socket.getfqdn() else: + if logger: + logger.debug("Will use 'socket.gethostname' " + "to get the system name") + return socket.gethostname() From b13f85f75d27862b5890585e6a3597b4f2091417 Mon Sep 17 00:00:00 2001 From: rafael Date: Fri, 11 Mar 2022 18:29:55 +0100 Subject: [PATCH 07/12] fix bug --- reframe/core/config.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/reframe/core/config.py b/reframe/core/config.py index 1289aec516..3ac4b3b732 100644 --- a/reframe/core/config.py +++ b/reframe/core/config.py @@ -66,6 +66,11 @@ def __init__(self, site_config, filename): self._subconfigs = {} self._local_system = None self._sticky_options = {} + self.autodetect_opts = { + 'autodetect_method': 'hostname', + 'autodetect_xthostname': True, + 'autodetect_fqdn': True, + } # Open and store the JSON schema for later validation schema_filename = os.path.join(reframe.INSTALL_PREFIX, 'reframe', @@ -222,7 +227,8 @@ def create(cls, filename, autodetect_opts): else: raise ConfigError(f"unknown configuration file type: '{filename}'") - ret.autodetect_opts = autodetect_opts + if autodetect_opts: + ret.autodetect_opts = autodetect_opts return ret @@ -620,13 +626,6 @@ def _find_config_file(): def load_config(filename=None, autodetect_opts=None): - if not autodetect_opts: - autodetect_opts = { - 'autodetect_method': 'hostname', - 'autodetect_xthostname': True, - 'autodetect_fqdn': True, - } - if filename is None: filename = _find_config_file() if filename is None: From 348df8d68c29291e2925830cc77dad165eb0edd0 Mon Sep 17 00:00:00 2001 From: rafael Date: Wed, 30 Mar 2022 13:39:03 +0200 Subject: [PATCH 08/12] fix comments --- reframe/core/config.py | 15 ++++++++------- reframe/frontend/cli.py | 37 +++++++++++++++++++++---------------- 2 files changed, 29 insertions(+), 23 deletions(-) diff --git a/reframe/core/config.py b/reframe/core/config.py index 3ac4b3b732..8cbac6c36c 100644 --- a/reframe/core/config.py +++ b/reframe/core/config.py @@ -66,8 +66,7 @@ def __init__(self, site_config, filename): self._subconfigs = {} self._local_system = None self._sticky_options = {} - self.autodetect_opts = { - 'autodetect_method': 'hostname', + self._autodetect_opts = { 'autodetect_xthostname': True, 'autodetect_fqdn': True, } @@ -218,7 +217,7 @@ def subconfig_system(self): return self._local_system @classmethod - def create(cls, filename, autodetect_opts): + def create(cls, filename, autodetect_method, **autodetect_opts): _, ext = os.path.splitext(filename) if ext == '.py': ret = cls._create_from_python(filename) @@ -227,8 +226,9 @@ def create(cls, filename, autodetect_opts): else: raise ConfigError(f"unknown configuration file type: '{filename}'") + ret._autodetect_method = autodetect_method if autodetect_opts: - ret.autodetect_opts = autodetect_opts + ret._autodetect_opts = autodetect_opts return ret @@ -273,7 +273,7 @@ def _create_from_json(cls, filename): def _detect_system(self): getlogger().debug('Detecting system') - hostname = get_hostname_cmd(self.autodetect_opts, getlogger()) + hostname = get_hostname_cmd(self._autodetect_opts, getlogger()) getlogger().debug( f'Looking for a matching configuration entry ' f'for system {hostname!r}' @@ -625,7 +625,8 @@ def _find_config_file(): return None -def load_config(filename=None, autodetect_opts=None): +def load_config(filename=None, autodetect_method='hostname', + **autodetect_opts): if filename is None: filename = _find_config_file() if filename is None: @@ -635,4 +636,4 @@ def load_config(filename=None, autodetect_opts=None): return _SiteConfig(settings.site_configuration, '') getlogger().debug(f'Loading configuration file: {filename!r}') - return _SiteConfig.create(filename, autodetect_opts) + return _SiteConfig.create(filename, autodetect_method, **autodetect_opts) diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index 5174ede43b..dcfd1afa14 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -8,6 +8,7 @@ import json import os import shlex +import socket import sys import time import traceback @@ -33,7 +34,6 @@ from reframe.frontend.executors.policies import (SerialExecutionPolicy, AsynchronousExecutionPolicy) from reframe.frontend.executors import Runner, generate_testcases -from reframe.utility import get_hostname_cmd def format_env(envvars): @@ -523,13 +523,6 @@ def main(): ) # Options not associated with command-line arguments - argparser.add_argument( - dest='use_login_shell', - envvar='RFM_USE_LOGIN_SHELL', - configvar='general/use_login_shell', - action='store_true', - help='Use a login shell for job scripts' - ) argparser.add_argument( dest='autodetect_method', envvar='RFM_AUTODETECT_METHOD', @@ -539,13 +532,13 @@ def main(): argparser.add_argument( dest='autodetect_fqdn', envvar='RFM_AUTODETECT_FQDN', - action='store_true', + action='store_false', help='Use FQDN as host name' ) argparser.add_argument( dest='autodetect_xthostname', envvar='RFM_AUTODETECT_XTHOSTNAME', - action='store_false', + action='store_true', default=True, help="Use Cray's xthostname file to find the host name" ) @@ -633,6 +626,13 @@ def main(): action='store_true', help='Trap job errors in job scripts and fail tests automatically' ) + argparser.add_argument( + dest='use_login_shell', + envvar='RFM_USE_LOGIN_SHELL', + configvar='general/use_login_shell', + action='store_true', + help='Use a login shell for job scripts' + ) def restrict_logging(): '''Restrict logging to errors only. @@ -660,7 +660,6 @@ def restrict_logging(): sys.exit(1) autodetect_opts = { - 'autodetect_method': options.autodetect_method or 'hostname', 'autodetect_xthostname': (options.autodetect_xthostname if options.autodetect_xthostname is not None else True), @@ -673,7 +672,7 @@ def restrict_logging(): # to print pretty messages; logging will be reconfigured by user's # configuration later site_config = config.load_config( - os.path.join(reframe.INSTALL_PREFIX, 'reframe/core/settings.py'), + os.path.join(reframe.INSTALL_PREFIX, 'reframe/core/settings.py') ) site_config.select_subconfig('generic') options.update_config(site_config) @@ -708,7 +707,10 @@ def restrict_logging(): ) sys.exit(0) - if autodetect_opts['autodetect_method'] not in ['hostname']: + if not options.autodetect_method: + options.autodetect_method = 'hostname' + + if options.autodetect_method not in ['hostname']: printer.error('unknown autodetect method ' f"`{options.autodetect_method}': Exiting...") sys.exit(1) @@ -718,7 +720,8 @@ def restrict_logging(): try: printer.debug('Loading user configuration') site_config = config.load_config(options.config_file, - autodetect_opts) + options.autodetect_method, + autodetect_opts=autodetect_opts) except warnings.ReframeDeprecationWarning as e: printer.warning(e) converted = config.convert_old_config(options.config_file) @@ -726,7 +729,9 @@ def restrict_logging(): f"configuration file has been converted " f"to the new syntax here: '{converted}'" ) - site_config = config.load_config(converted, autodetect_opts) + site_config = config.load_config(converted, + options.autodetect_method, + autodetect_opts=autodetect_opts) site_config.validate() @@ -912,7 +917,7 @@ def print_infoline(param, value): 'cmdline': ' '.join(sys.argv), 'config_file': rt.site_config.filename, 'data_version': runreport.DATA_VERSION, - 'hostname': get_hostname_cmd(autodetect_opts), + 'hostname': socket.gethostname(), 'prefix_output': rt.output_prefix, 'prefix_stage': rt.stage_prefix, 'user': osext.osuser(), From def2b58b24ab68fc01a4edc9c2a45b82947dc142 Mon Sep 17 00:00:00 2001 From: Rafael Sarmiento Date: Wed, 30 Mar 2022 18:01:39 +0200 Subject: [PATCH 09/12] fix docs --- cscs-checks/microbenchmarks/cpu/dgemm/dgemm.py | 8 +------- docs/configure.rst | 3 ++- docs/manpage.rst | 8 +++----- 3 files changed, 6 insertions(+), 13 deletions(-) diff --git a/cscs-checks/microbenchmarks/cpu/dgemm/dgemm.py b/cscs-checks/microbenchmarks/cpu/dgemm/dgemm.py index 9448e58e84..b11a046b51 100644 --- a/cscs-checks/microbenchmarks/cpu/dgemm/dgemm.py +++ b/cscs-checks/microbenchmarks/cpu/dgemm/dgemm.py @@ -8,19 +8,13 @@ @rfm.simple_test -class DGEMMTest(rfm.RegressionTest): - descr = 'DGEMM performance test' - sourcepath = 'dgemm.c' - - # the perf patterns are automaticaly generated inside sanity - perf_patterns = {} +class dgemm_Test(rfm.RegressionTest): valid_systems = ['daint:gpu', 'daint:mc', 'dom:gpu', 'dom:mc', 'arolla:cn', 'arolla:pn', 'tsa:cn', 'tsa:pn', 'eiger:mc', 'pilatus:mc'] num_tasks = 0 use_multithreading = False executable_opts = ['6144', '12288', '3072'] - build_system = 'SingleSource' arch_refs = { 'haswell@12c': (300.0, -0.15, None, 'Gflop/s'), 'broadwell@36c': (1040.0, -0.15, None, 'Gflop/s'), diff --git a/docs/configure.rst b/docs/configure.rst index fe7221dc72..2146c29147 100644 --- a/docs/configure.rst +++ b/docs/configure.rst @@ -239,7 +239,8 @@ When launched, ReFrame will pick the first matching configuration and load it. The way in which the system name is determined can be controlled by the environment variables ``RFM_AUTODETECT_METHOD``, ``RFM_AUTODETECT_FQDN`` and ``RFM_AUTODETECT_XTHOSTNAME``. ``RFM_AUTODETECT_METHOD`` is a string that specifies the method used to detect the system name. Its default value is ``'hostname'``, which currently is the only supported option. -The host name will be determined by ``socket.gethostname()``, unless the boolean ``RFM_AUTODETECT_FQDN`` is set, which make ReFrame use ``socket.getfqdn()`` instead. +By default, the host name is determined by ``socket.getfqdn()``. +Setting the boolean ``RFM_AUTODETECT_FQDN`` to ``false`` makes ReFrame use ``socket.gethostname()`` instead. The last option, ``RFM_AUTODETECT_XTHOSTNAME`` is a boolean and when set, ReFrame first tries to obtain the hostname from ``/etc/xthostname``, which provides the unqualified *machine name* in Cray systems. If this cannot be found, ReFrame falls back to the one of the two options set by ``RFM_AUTODETECT_FQDN``. diff --git a/docs/manpage.rst b/docs/manpage.rst index 3c66891344..4f2a6ad4ac 100644 --- a/docs/manpage.rst +++ b/docs/manpage.rst @@ -1026,9 +1026,7 @@ Here is an alphabetical list of the environment variables recognized by ReFrame: Method to detect the system. - There is only one method defined - - - ``hostname``: The system will be detected by finding its hostname. + - ``hostname``: The `hostname` command will be used to detect the current system. .. table:: :align: left @@ -1044,7 +1042,7 @@ Here is an alphabetical list of the environment variables recognized by ReFrame: .. envvar:: RFM_AUTODETECT_FQDN - Use FQDN as host name. + Use the fully qualified domain name as the host name. .. table:: :align: left @@ -1060,7 +1058,7 @@ Here is an alphabetical list of the environment variables recognized by ReFrame: .. envvar:: RFM_AUTODETECT_XTHOSTNAME - Use Cray's xthostname file to find the host name. + If present, use Cray's `/etc/xthostname` file to retrieve the current system's name .. table:: :align: left From e4681720958f52ba87d554406427f9c2d891d1fd Mon Sep 17 00:00:00 2001 From: Rafael Sarmiento Date: Wed, 30 Mar 2022 18:59:53 +0200 Subject: [PATCH 10/12] remove dgemm file --- cscs-checks/microbenchmarks/cpu/dgemm/dgemm.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/cscs-checks/microbenchmarks/cpu/dgemm/dgemm.py b/cscs-checks/microbenchmarks/cpu/dgemm/dgemm.py index b11a046b51..9448e58e84 100644 --- a/cscs-checks/microbenchmarks/cpu/dgemm/dgemm.py +++ b/cscs-checks/microbenchmarks/cpu/dgemm/dgemm.py @@ -8,13 +8,19 @@ @rfm.simple_test -class dgemm_Test(rfm.RegressionTest): +class DGEMMTest(rfm.RegressionTest): + descr = 'DGEMM performance test' + sourcepath = 'dgemm.c' + + # the perf patterns are automaticaly generated inside sanity + perf_patterns = {} valid_systems = ['daint:gpu', 'daint:mc', 'dom:gpu', 'dom:mc', 'arolla:cn', 'arolla:pn', 'tsa:cn', 'tsa:pn', 'eiger:mc', 'pilatus:mc'] num_tasks = 0 use_multithreading = False executable_opts = ['6144', '12288', '3072'] + build_system = 'SingleSource' arch_refs = { 'haswell@12c': (300.0, -0.15, None, 'Gflop/s'), 'broadwell@36c': (1040.0, -0.15, None, 'Gflop/s'), From 7c570eea9ed3a594cc0f9e49637bf22a8d07d741 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Fri, 1 Apr 2022 13:28:16 +0200 Subject: [PATCH 11/12] Improve implementation - Support defaults in ArgumentParser for environment variable-only options - Remove `get_hostname_cmd()` - Autodetection happens entirely in `_SiteConfig` - New method for setting the autodetection method in `_SiteConfig`. --- docs/configure.rst | 15 +++--- docs/manpage.rst | 100 +++++++++++++++++++---------------- reframe/core/config.py | 61 +++++++++++++++------ reframe/frontend/argparse.py | 7 ++- reframe/frontend/cli.py | 40 +++++++------- reframe/utility/__init__.py | 24 --------- unittests/test_argparser.py | 19 +++++++ unittests/test_config.py | 14 +++++ 8 files changed, 163 insertions(+), 117 deletions(-) diff --git a/docs/configure.rst b/docs/configure.rst index 2146c29147..d50ec69724 100644 --- a/docs/configure.rst +++ b/docs/configure.rst @@ -236,17 +236,14 @@ Picking a System Configuration As discussed previously, ReFrame's configuration file can store the configurations for multiple systems. When launched, ReFrame will pick the first matching configuration and load it. -The way in which the system name is determined can be controlled by the environment variables ``RFM_AUTODETECT_METHOD``, ``RFM_AUTODETECT_FQDN`` and ``RFM_AUTODETECT_XTHOSTNAME``. -``RFM_AUTODETECT_METHOD`` is a string that specifies the method used to detect the system name. -Its default value is ``'hostname'``, which currently is the only supported option. -By default, the host name is determined by ``socket.getfqdn()``. -Setting the boolean ``RFM_AUTODETECT_FQDN`` to ``false`` makes ReFrame use ``socket.gethostname()`` instead. -The last option, ``RFM_AUTODETECT_XTHOSTNAME`` is a boolean and when set, ReFrame first tries to obtain the hostname from ``/etc/xthostname``, which provides the unqualified *machine name* in Cray systems. -If this cannot be found, ReFrame falls back to the one of the two options set by ``RFM_AUTODETECT_FQDN``. - -Having retrieved the hostname, ReFrame goes through all the systems in its configuration and tries to match the hostname against any of the patterns defined in each system's ``hostnames`` property. +ReFrame uses an auto-detection mechanism to get information about the host it is running on and uses that information to pick the right system configuration. +Currently, only one auto-detection method is supported that retrieves the hostname. +Based on this, ReFrame goes through all the systems in its configuration and tries to match the hostname against any of the patterns defined in each system's ``hostnames`` property. The detection process stops at the first match found, and that system's configuration is selected. +The auto-detection process can be controlled through the :envvar:`RFM_AUTODETECT_METHOD`, :envvar:`RFM_AUTODETECT_FQDN` and :envvar:`RFM_AUTODETECT_XTHOSTNAME` environment variables. + + As soon as a system configuration is selected, all configuration objects that have a ``target_systems`` property are resolved against the selected system, and any configuration object that is not applicable is dropped. So, internally, ReFrame keeps an *instantiation* of the site configuration for the selected system only. To better understand this, let's assume that we have the following ``environments`` defined: diff --git a/docs/manpage.rst b/docs/manpage.rst index 4f2a6ad4ac..e653954607 100644 --- a/docs/manpage.rst +++ b/docs/manpage.rst @@ -940,138 +940,146 @@ Boolean environment variables can have any value of ``true``, ``yes``, ``y`` (ca Here is an alphabetical list of the environment variables recognized by ReFrame: -.. envvar:: RFM_CHECK_SEARCH_PATH +.. envvar:: RFM_AUTODETECT_FQDN + + Use the fully qualified domain name as the hostname. + This is a boolean variable and defaults to ``1``. - A colon-separated list of filesystem paths where ReFrame should search for tests. .. table:: :align: left ================================== ================== - Associated command line option :option:`-c` - Associated configuration parameter :js:attr:`check_search_path` general configuration parameter + Associated command line option N/A + Associated configuration parameter N/A ================================== ================== -.. envvar:: RFM_CHECK_SEARCH_RECURSIVE + .. versionadded:: 3.11.0 - Search for test files recursively in directories found in the check search path. + +.. envvar:: RFM_AUTODETECT_METHOD + + Method to use for detecting the current system and pick the right configuration. + The following values can be used: + + - ``hostname``: The ``hostname`` command will be used to detect the current system. + This is the default value, if not specified. .. table:: :align: left ================================== ================== - Associated command line option :option:`-R` - Associated configuration parameter :js:attr:`check_search_recursive` general configuration parameter + Associated command line option N/A + Associated configuration parameter N/A ================================== ================== -.. envvar:: RFM_CLEAN_STAGEDIR + .. versionadded:: 3.11.0 - Clean stage directory of tests before populating it. - .. versionadded:: 3.1 +.. envvar:: RFM_AUTODETECT_XTHOSTNAME + + Use ``/etc/xthostname`` file, if present, to retrieve the current system's name. + If the file cannot be found, the hostname will be retrieved using the ``hostname`` command. + This is a boolean variable and defaults to ``1``. + + This option meaningful for Cray systems. .. table:: :align: left ================================== ================== - Associated command line option :option:`--dont-restage` - Associated configuration parameter :js:attr:`clean_stagedir` general configuration parameter + Associated command line option N/A + Associated configuration parameter N/A ================================== ================== -.. envvar:: RFM_COLORIZE + .. versionadded:: 3.11.0 - Enable output coloring. + +.. envvar:: RFM_CHECK_SEARCH_PATH + + A colon-separated list of filesystem paths where ReFrame should search for tests. .. table:: :align: left ================================== ================== - Associated command line option :option:`--nocolor` - Associated configuration parameter :js:attr:`colorize` general configuration parameter + Associated command line option :option:`-c` + Associated configuration parameter :js:attr:`check_search_path` general configuration parameter ================================== ================== -.. envvar:: RFM_COMPACT_TEST_NAMES +.. envvar:: RFM_CHECK_SEARCH_RECURSIVE - Enable the new test naming scheme. + Search for test files recursively in directories found in the check search path. .. table:: :align: left ================================== ================== - Associated command line option N/A - Associated configuration parameter :js:attr:`compact_test_names` general configuration parameter + Associated command line option :option:`-R` + Associated configuration parameter :js:attr:`check_search_recursive` general configuration parameter ================================== ================== - .. versionadded:: 3.9.0 +.. envvar:: RFM_CLEAN_STAGEDIR -.. envvar:: RFM_CONFIG_FILE + Clean stage directory of tests before populating it. - Set the configuration file for ReFrame. + .. versionadded:: 3.1 .. table:: :align: left ================================== ================== - Associated command line option :option:`-C` - Associated configuration parameter N/A + Associated command line option :option:`--dont-restage` + Associated configuration parameter :js:attr:`clean_stagedir` general configuration parameter ================================== ================== -.. envvar:: RFM_AUTODETECT_METHOD - - Method to detect the system. +.. envvar:: RFM_COLORIZE - - ``hostname``: The `hostname` command will be used to detect the current system. + Enable output coloring. .. table:: :align: left ================================== ================== - Associated command line option N/A - Associated configuration parameter N/A + Associated command line option :option:`--nocolor` + Associated configuration parameter :js:attr:`colorize` general configuration parameter ================================== ================== - .. versionadded:: 3.12.0 - - -.. envvar:: RFM_AUTODETECT_FQDN +.. envvar:: RFM_COMPACT_TEST_NAMES - Use the fully qualified domain name as the host name. + Enable the new test naming scheme. .. table:: :align: left ================================== ================== Associated command line option N/A - Associated configuration parameter N/A + Associated configuration parameter :js:attr:`compact_test_names` general configuration parameter ================================== ================== - - .. versionadded:: 3.12.0 + .. versionadded:: 3.9.0 -.. envvar:: RFM_AUTODETECT_XTHOSTNAME +.. envvar:: RFM_CONFIG_FILE - If present, use Cray's `/etc/xthostname` file to retrieve the current system's name + Set the configuration file for ReFrame. .. table:: :align: left ================================== ================== - Associated command line option N/A + Associated command line option :option:`-C` Associated configuration parameter N/A ================================== ================== - .. versionadded:: 3.12.0 - - .. envvar:: RFM_GIT_TIMEOUT Timeout value in seconds used when checking if a git repository exists. diff --git a/reframe/core/config.py b/reframe/core/config.py index 8cbac6c36c..18ab2cd11e 100644 --- a/reframe/core/config.py +++ b/reframe/core/config.py @@ -11,6 +11,7 @@ import jsonschema import os import re +import socket import tempfile import reframe @@ -19,7 +20,7 @@ from reframe.core.environments import normalize_module_list from reframe.core.exceptions import ConfigError, ReframeFatalError from reframe.core.logging import getlogger -from reframe.utility import ScopedDict, get_hostname_cmd +from reframe.utility import ScopedDict def _match_option(opt, opt_map): @@ -59,6 +60,26 @@ def _get(site_config, option, *args, **kwargs): return _do_normalize +def _hostname(use_fqdn, use_xthostname): + '''Return hostname''' + if use_xthostname: + try: + xthostname_file = '/etc/xthostname' + getlogger().debug(f'Trying {xthostname_file!r}...') + with open(xthostname_file) as fp: + return fp.read() + except OSError as e: + '''Log the error and continue to the next method''' + getlogger().debug(f'Failed to read {xthostname_file!r}') + + if use_fqdn: + getlogger().debug('Using FQDN...') + return socket.getfqdn() + + getlogger().debug('Using standard hostname...') + return socket.gethostname() + + class _SiteConfig: def __init__(self, site_config, filename): self._site_config = copy.deepcopy(site_config) @@ -66,9 +87,12 @@ def __init__(self, site_config, filename): self._subconfigs = {} self._local_system = None self._sticky_options = {} + self._autodetect_meth = 'hostname' self._autodetect_opts = { - 'autodetect_xthostname': True, - 'autodetect_fqdn': True, + 'hostname': { + 'use_fqdn': False, + 'use_xthostname': False, + } } # Open and store the JSON schema for later validation @@ -107,6 +131,15 @@ def __getitem__(self, key): def __getattr__(self, attr): return getattr(self._pick_config(), attr) + def set_autodetect_meth(self, method, **opts): + self._autodetect_meth = method + try: + self._autodetect_opts[method].update(opts) + except KeyError: + raise ConfigError( + f'unknown auto-detection method: {method!r}' + ) from None + @property def schema(self): '''Configuration schema''' @@ -217,7 +250,7 @@ def subconfig_system(self): return self._local_system @classmethod - def create(cls, filename, autodetect_method, **autodetect_opts): + def create(cls, filename): _, ext = os.path.splitext(filename) if ext == '.py': ret = cls._create_from_python(filename) @@ -226,10 +259,6 @@ def create(cls, filename, autodetect_method, **autodetect_opts): else: raise ConfigError(f"unknown configuration file type: '{filename}'") - ret._autodetect_method = autodetect_method - if autodetect_opts: - ret._autodetect_opts = autodetect_opts - return ret @classmethod @@ -272,12 +301,15 @@ def _create_from_json(cls, filename): return _SiteConfig(config, filename) def _detect_system(self): - getlogger().debug('Detecting system') - hostname = get_hostname_cmd(self._autodetect_opts, getlogger()) getlogger().debug( - f'Looking for a matching configuration entry ' - f'for system {hostname!r}' + f'Detecting system using method: {self._autodetect_meth!r}' + ) + hostname = _hostname( + self._autodetect_opts[self._autodetect_meth]['use_fqdn'], + self._autodetect_opts[self._autodetect_meth]['use_xthostname'], ) + getlogger().debug(f'Retrieved hostname: {hostname!r}') + getlogger().debug(f'Looking for a matching configuration entry') for system in self._site_config['systems']: for patt in system['hostnames']: if re.match(patt, hostname): @@ -625,8 +657,7 @@ def _find_config_file(): return None -def load_config(filename=None, autodetect_method='hostname', - **autodetect_opts): +def load_config(filename=None): if filename is None: filename = _find_config_file() if filename is None: @@ -636,4 +667,4 @@ def load_config(filename=None, autodetect_method='hostname', return _SiteConfig(settings.site_configuration, '') getlogger().debug(f'Loading configuration file: {filename!r}') - return _SiteConfig.create(filename, autodetect_method, **autodetect_opts) + return _SiteConfig.create(filename) diff --git a/reframe/frontend/argparse.py b/reframe/frontend/argparse.py index d88de38f88..68d4894e65 100644 --- a/reframe/frontend/argparse.py +++ b/reframe/frontend/argparse.py @@ -82,7 +82,7 @@ def __getattr__(self, name): if name not in self.__option_map: return ret - envvar, _, action, arg_type = self.__option_map[name] + envvar, _, action, arg_type, default = self.__option_map[name] if ret is None and envvar is not None: # Try the environment variable envvar, *delim = envvar.split(maxsplit=2) @@ -107,6 +107,8 @@ def __getattr__(self, name): f'cannot convert environment variable {envvar!r} ' f'to {arg_type.__name__!r}' ) from err + else: + ret = default return ret @@ -183,7 +185,8 @@ def add_argument(self, *flags, **kwargs): kwargs.get('envvar', None), kwargs.get('configvar', None), kwargs.get('action', 'store'), - kwargs.get('type', str) + kwargs.get('type', str), + kwargs.get('default', None) ) # Remove envvar and configvar keyword arguments and force dest # argument, even if we guessed it, in order to guard against changes diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index dcfd1afa14..ada50fa4e8 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -27,6 +27,7 @@ import reframe.frontend.runreport as runreport import reframe.utility.jsonext as jsonext import reframe.utility.osext as osext +import reframe.utility.typecheck as typ from reframe.frontend.printer import PrettyPrinter @@ -523,23 +524,27 @@ def main(): ) # Options not associated with command-line arguments + argparser.add_argument( + dest='autodetect_fqdn', + envvar='RFM_AUTODETECT_FQDN', + action='store', + default=True, + type=typ.Bool, + help='Use FQDN as host name' + ) argparser.add_argument( dest='autodetect_method', envvar='RFM_AUTODETECT_METHOD', action='store', + default='hostname', help='Method to detect the system' ) - argparser.add_argument( - dest='autodetect_fqdn', - envvar='RFM_AUTODETECT_FQDN', - action='store_false', - help='Use FQDN as host name' - ) argparser.add_argument( dest='autodetect_xthostname', envvar='RFM_AUTODETECT_XTHOSTNAME', - action='store_true', + action='store', default=True, + type=typ.Bool, help="Use Cray's xthostname file to find the host name" ) argparser.add_argument( @@ -707,21 +712,11 @@ def restrict_logging(): ) sys.exit(0) - if not options.autodetect_method: - options.autodetect_method = 'hostname' - - if options.autodetect_method not in ['hostname']: - printer.error('unknown autodetect method ' - f"`{options.autodetect_method}': Exiting...") - sys.exit(1) - # Now configure ReFrame according to the user configuration file try: try: printer.debug('Loading user configuration') - site_config = config.load_config(options.config_file, - options.autodetect_method, - autodetect_opts=autodetect_opts) + site_config = config.load_config(options.config_file) except warnings.ReframeDeprecationWarning as e: printer.warning(e) converted = config.convert_old_config(options.config_file) @@ -729,11 +724,14 @@ def restrict_logging(): f"configuration file has been converted " f"to the new syntax here: '{converted}'" ) - site_config = config.load_config(converted, - options.autodetect_method, - autodetect_opts=autodetect_opts) + site_config = config.load_config(converted) site_config.validate() + site_config.set_autodetect_meth( + options.autodetect_method, + use_fqdn=options.autodetect_fqdn, + use_xthostname=options.autodetect_xthostname + ) # We ignore errors about unresolved sections or configuration # parameters here, because they might be defined at the individual diff --git a/reframe/utility/__init__.py b/reframe/utility/__init__.py index 70da820528..2fa3255ac1 100644 --- a/reframe/utility/__init__.py +++ b/reframe/utility/__init__.py @@ -13,7 +13,6 @@ import itertools import os import re -import socket import sys import types import weakref @@ -25,29 +24,6 @@ from . import typecheck as typ -def get_hostname_cmd(autodetect_opts, logger=None): - if autodetect_opts['autodetect_xthostname']: - if os.path.exists('/etc/xthostname'): - # Get the cluster name on Cray systems - with open('/etc/xthostname') as fp: - return fp.read() - - elif logger: - logger.debug("Did not find '/etc/xthostname')") - - if autodetect_opts['autodetect_fqdn']: - if logger: - logger.debug("Will use 'socket.getfqdn' to get the system name") - - return socket.getfqdn() - else: - if logger: - logger.debug("Will use 'socket.gethostname' " - "to get the system name") - - return socket.gethostname() - - def seconds_to_hms(seconds): '''Convert time in seconds to hours, minutes and seconds. diff --git a/unittests/test_argparser.py b/unittests/test_argparser.py index ea7d892914..032490a04a 100644 --- a/unittests/test_argparser.py +++ b/unittests/test_argparser.py @@ -107,6 +107,14 @@ def extended_parser(): '--git-timeout', envvar='RFM_GIT_TIMEOUT', action='store', configvar='general/git_timeout', type=float ) + + # Option that is only associated with an environment variable + parser.add_argument( + dest='env_option', + envvar='RFM_ENV_OPT', + action='store', + default='bar' + ) foo_options.add_argument( '--timestamp', action='store', envvar='RFM_TIMESTAMP_DIRS', configvar='general/timestamp_dirs' @@ -187,3 +195,14 @@ def test_option_envvar_conversion_error(default_exec_ctx, extended_parser): options = extended_parser.parse_args(['--nocolor']) errors = options.update_config(site_config) assert len(errors) == 2 + + +def test_envvar_option(default_exec_ctx, extended_parser): + with rt.temp_environment(variables={'RFM_ENV_OPT': 'BAR'}): + options = extended_parser.parse_args([]) + assert options.env_option == 'BAR' + + +def test_envvar_option_default_val(default_exec_ctx, extended_parser): + options = extended_parser.parse_args([]) + assert options.env_option == 'bar' diff --git a/unittests/test_config.py b/unittests/test_config.py index 88331ac953..c68a08e504 100644 --- a/unittests/test_config.py +++ b/unittests/test_config.py @@ -387,3 +387,17 @@ def test_system_create(): assert partition.processor.num_cores_per_socket == 4 assert partition.processor.num_numa_nodes == 1 assert partition.processor.num_cores_per_numa_node == 4 + + +def test_hostname_autodetection(): + # This exercises only the various execution paths + + # We set the autodetection method and we call `select_subconfig()` in + # order to trigger the auto-detection + site_config = config.load_config('unittests/resources/settings.py') + for use_xthostname in (True, False): + for use_fqdn in (True, False): + site_config.set_autodetect_meth('hostname', + use_fqdn=use_fqdn, + use_xthostname=use_xthostname) + site_config.select_subconfig() From f73ecf9dc78d702ffeb25494a2b256602f391298 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Fri, 1 Apr 2022 13:42:36 +0200 Subject: [PATCH 12/12] Remove stale code --- reframe/core/config.py | 6 ++---- reframe/frontend/cli.py | 9 --------- 2 files changed, 2 insertions(+), 13 deletions(-) diff --git a/reframe/core/config.py b/reframe/core/config.py index 18ab2cd11e..6750696324 100644 --- a/reframe/core/config.py +++ b/reframe/core/config.py @@ -253,14 +253,12 @@ def subconfig_system(self): def create(cls, filename): _, ext = os.path.splitext(filename) if ext == '.py': - ret = cls._create_from_python(filename) + return cls._create_from_python(filename) elif ext == '.json': - ret = cls._create_from_json(filename) + return cls._create_from_json(filename) else: raise ConfigError(f"unknown configuration file type: '{filename}'") - return ret - @classmethod def _create_from_python(cls, filename): try: diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index ada50fa4e8..2382468d61 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -664,15 +664,6 @@ def restrict_logging(): argparser.print_help() sys.exit(1) - autodetect_opts = { - 'autodetect_xthostname': (options.autodetect_xthostname - if options.autodetect_xthostname is not None - else True), - 'autodetect_fqdn': (options.autodetect_fqdn - if options.autodetect_fqdn is not None - else True), - } - # First configure logging with our generic configuration so as to be able # to print pretty messages; logging will be reconfigured by user's # configuration later