Merge pull request #2832 from ekouts/feat/system_autodetect_methods

[feat] Support custom system auto-detection methods

docs/config_reference.rst

@@ -61,6 +61,29 @@ It consists of the following properties, which we also call conventionally *conf
    A list of `general configuration objects <#general-configuration>`__.
 
 
+.. py:data:: autodetect_methods
+
+   :required: No
+   :default: ``["py::socket.gethostname"]``
+
+   A list of system auto-detection methods for identifying the current system.
+
+   The list can contain two types of methods:
+
+   1. Python methods: These are prefixed with ``py::`` and should point to a Python callable taking zero arguments and returning a string.
+      If the specified Python callable is not prefixed with a module, it will be looked up in the loaded configuration files starting from the last file.
+      If the requested symbol cannot be found, a warning will be issued and the method will be ignored.
+   2. Shell commands: Any string not prefixed with ``py::`` will be treated as a shell command and will be executed *during auto-detection* to retrieve the hostname.
+      The standard output of the command will be used.
+
+   If the :option:`--system` option is not passed, ReFrame will try to autodetect the current system trying the methods in this list successively, until one of them succeeds.
+   The resulting name will be matched against the :attr:`~config.systems.hostnames` patterns of each system and the system that matches first will be used as the current one.
+
+   The auto-detection methods can also be controlled through the :envvar:`RFM_AUTODETECT_METHODS` environment variable.
+
+   .. versionadded:: 4.3
+
+
 .. warning::
    .. versionchanged:: 4.0.0
       The :data:`schedulers` section is removed.

docs/configure.rst

@@ -311,12 +311,9 @@ As discussed previously, ReFrame's configuration file can store the configuratio
 When launched, ReFrame will pick the first matching configuration and load it.
 
 ReFrame uses an auto-detection mechanism to get information about the host it is running on and uses that information to pick the right system configuration.
-Currently, only one auto-detection method is supported that retrieves the hostname.
-Based on this, ReFrame goes through all the systems in its configuration and tries to match the hostname against any of the patterns defined in each system's ``hostnames`` property.
-The detection process stops at the first match found, and that system's configuration is selected.
-
-The auto-detection process can be controlled through the :envvar:`RFM_AUTODETECT_METHOD`, :envvar:`RFM_AUTODETECT_FQDN` and :envvar:`RFM_AUTODETECT_XTHOSTNAME` environment variables.
-
+The default auto-detection method uses the ``hostname`` command, but you can define more methods by setting either the :attr:`autodetect_methods` configuration parameter or the :envvar:`RFM_AUTODETECT_METHODS` environment variable.
+After having retrieved the hostname, ReFrame goes through all the systems in its configuration and tries to match it against the :attr:`~config.systems.hostnames` patterns defined for each system.
+The first system whose :attr:`~config.systems.hostnames` match will become the current system and its configuration will be loaded.
 
 As soon as a system configuration is selected, all configuration objects that have a ``target_systems`` property are resolved against the selected system, and any configuration object that is not applicable is dropped.
 So, internally, ReFrame keeps an *instantiation* of the site configuration for the selected system only.

docs/manpage.rst

@@ -1136,6 +1136,9 @@ Whenever an environment variable is associated with a configuration option, its
    .. versionchanged:: 4.0.0
       This variable now defaults to ``0``.
 
+   .. deprecated:: 4.3
+      Please use ``RFM_AUTODETECT_METHODS=py::fqdn`` in the future.
+
 
 .. envvar:: RFM_AUTODETECT_METHOD
 
@@ -1155,6 +1158,16 @@ Whenever an environment variable is associated with a configuration option, its
 
 
    .. versionadded:: 3.11.0
+   .. deprecated:: 4.3
+      This has no effect.
+      For setting multiple auto-detection methods, please use the :envvar:`RFM_AUTODETECT_METHODS`.
+
+.. envvar:: RFM_AUTODETECT_METHODS
+
+   A comma-separated list of system auto-detection methods.
+   Please refer to the :attr:`autodetect_methods` configuration parameter for more information on how to set this variable.
+
+   .. versionadded:: 4.3
 
 
 .. envvar:: RFM_AUTODETECT_XTHOSTNAME
@@ -1179,6 +1192,10 @@ Whenever an environment variable is associated with a configuration option, its
    .. versionchanged:: 4.0.0
       This variable now defaults to ``0``.
 
+   .. deprecated:: 4.3
+      Please use ``RFM_AUTODETECT_METHODS='cat /etc/xthostname,hostname'`` in the future.
+
+
 .. envvar:: RFM_CHECK_SEARCH_PATH
 
    A colon-separated list of filesystem paths where ReFrame should search for tests.

reframe/core/config.py

@@ -7,18 +7,19 @@
 import copy
 import fnmatch
 import functools
+import importlib
 import itertools
 import json
 import jsonschema
 import os
 import re
-import socket
 
 import reframe
 import reframe.core.settings as settings
 import reframe.utility as util
+import reframe.utility.osext as osext
 from reframe.core.environments import normalize_module_list
-from reframe.core.exceptions import ConfigError, ReframeFatalError
+from reframe.core.exceptions import (ConfigError, ReframeFatalError)
 from reframe.core.logging import getlogger
 from reframe.utility import ScopedDict
 
@@ -60,40 +61,15 @@ def _get(site_config, option, *args, **kwargs):
     return _do_normalize
 
 
-def _hostname(use_fqdn, use_xthostname):
-    '''Return hostname'''
-    if use_xthostname:
-        try:
-            xthostname_file = '/etc/xthostname'
-            getlogger().debug(f'Trying {xthostname_file!r}...')
-            with open(xthostname_file) as fp:
-                return fp.read()
-        except OSError as e:
-            '''Log the error and continue to the next method'''
-            getlogger().debug(f'Failed to read {xthostname_file!r}')
-
-    if use_fqdn:
-        getlogger().debug('Using FQDN...')
-        return socket.getfqdn()
-
-    getlogger().debug('Using standard hostname...')
-    return socket.gethostname()
-
-
 class _SiteConfig:
     def __init__(self):
         self._site_config = None
+        self._config_modules = []
         self._sources = []
         self._subconfigs = {}
         self._local_system = None
         self._sticky_options = {}
-        self._autodetect_meth = 'hostname'
-        self._autodetect_opts = {
-            'hostname': {
-                'use_fqdn': False,
-                'use_xthostname': False,
-            }
-        }
+        self._autodetect_methods = []
         self._definitions = {
             'systems': {},
             'partitions': {},
@@ -195,7 +171,7 @@ def update_config(self, config, filename):
         nc = copy.deepcopy(config)
         if self._site_config is None:
             self._site_config = nc
-            return self
+            return
 
         mergeable_sections = ('general', 'logging', 'schedulers')
         for sec in nc.keys():
@@ -206,10 +182,11 @@ def update_config(self, config, filename):
                     self._site_config[sec], nc[sec]
                 )
             else:
-                if sec == 'systems':
-                    # Systems have to be inserted in the beginning of the list,
-                    # since they are selected by the first matching entry in
-                    # `hostnames`.
+                if sec in ('systems', 'autodetect_methods'):
+                    # Systems have to be inserted in the beginning of the
+                    # list, since they are selected by the first matching
+                    # entry in `hostnames`. Similarly, the autodetection
+                    # methods are tried from left to right.
                     self._site_config[sec] = nc[sec] + self._site_config[sec]
                 else:
                     self._site_config[sec] += nc[sec]
@@ -239,14 +216,8 @@ def __getitem__(self, key):
     def __getattr__(self, attr):
         return getattr(self._pick_config(), attr)
 
-    def set_autodetect_meth(self, method, **opts):
-        self._autodetect_meth = method
-        try:
-            self._autodetect_opts[method].update(opts)
-        except KeyError:
-            raise ConfigError(
-                f'unknown auto-detection method: {method!r}'
-            ) from None
+    def set_autodetect_methods(self, methods):
+        self._site_config['autodetect_methods'] = list(methods)
 
     @property
     def schema(self):
@@ -364,22 +335,15 @@ def load_config_python(self, filename):
             # import_module_from_file() may raise an ImportError if the
             # configuration file is under ReFrame's top-level directory
             raise ConfigError(
-                f"could not load Python configuration file: '{filename}'"
+                f'could not load Python configuration file: {filename!r}'
             ) from e
 
-        if hasattr(mod, 'settings'):
-            # Looks like an old style config
-            raise ConfigError(
-                f"the syntax of the configuration file {filename!r} "
-                f"is no longer supported"
-            )
-
-        mod = util.import_module_from_file(filename)
         if not hasattr(mod, 'site_configuration'):
             raise ConfigError(
                 f"not a valid Python configuration file: '{filename}'"
             )
 
+        self._config_modules.append(mod)
         self.update_config(mod.site_configuration, filename)
 
     def load_config_json(self, filename):
@@ -393,14 +357,75 @@ def load_config_json(self, filename):
 
         self.update_config(config, filename)
 
-    def _detect_system(self):
-        getlogger().debug(
-            f'Detecting system using method: {self._autodetect_meth!r}'
-        )
-        hostname = _hostname(
-            self._autodetect_opts[self._autodetect_meth]['use_fqdn'],
-            self._autodetect_opts[self._autodetect_meth]['use_xthostname'],
+    def _setup_autodect_methods(self):
+        def _py_meth(m):
+            try:
+                module, symbol = m.rsplit('.', maxsplit=1)
+            except ValueError:
+                # Not enough values to unpack; we assume a single symbol
+                module, symbol = None, m
+
+            try:
+                if module:
+                    mod = importlib.import_module(module)
+                    return getattr(mod, symbol)
+
+                if not self._config_modules:
+                    raise ConfigError(
+                        f'no module context for requested symbol: {symbol!r}'
+                    )
+
+                # Symbol is local to one of the config files; try to resolve
+                # it in reverse order
+                for mod in reversed(self._config_modules):
+                    if hasattr(mod, symbol):
+                        return getattr(mod, symbol)
+
+                raise ConfigError(f'symbol {symbol!r} is not defined in '
+                                  f'any of the loaded configuration files')
+            except (AttributeError, ImportError, ConfigError) as e:
+                getlogger().warning(f"ignoring autodetection method 'py::{m}' "
+                                    f"due to errors: {e}")
+
+        def _sh_meth(m):
+            def _fn():
+                completed = osext.run_command(m, check=True)
+                return completed.stdout.strip()
+
+            return _fn
+
+        methods = self._site_config.get(
+            'autodetect_methods',
+            self._schema['defaults']['autodetect_methods']
         )
+        for m in methods:
+            if m.startswith('py::'):
+                fn = _py_meth(m.lstrip('py::'))
+                if fn is not None:
+                    self._autodetect_methods.append((m, fn))
+            else:
+                self._autodetect_methods.append((m, _sh_meth(m)))
+
+    def _detect_system(self):
+        getlogger().debug('Autodetecting system')
+        if not self._autodetect_methods:
+            self._setup_autodect_methods()
+
+        hostname = None
+        for meth, fn in self._autodetect_methods:
+            getlogger().debug(f'Trying autodetection method: {meth!r}')
+            try:
+                hostname = fn()
+            except Exception as e:
+                getlogger().debug(f'Autodetection method {meth!r} failed: {e}')
+            else:
+                break
+
+        if hostname is None:
+            raise ConfigError('all autodetection methods failed; '
+                              'try passing a system name explicitly using '
+                              'the `--system` option')
+
         getlogger().debug(f'Retrieved hostname: {hostname!r}')
         getlogger().debug(f'Looking for a matching configuration entry')
         for system in self._site_config['systems']:
@@ -412,8 +437,8 @@ def _detect_system(self):
                     )
                     return sysname
 
-        raise ConfigError(f"could not find a configuration entry "
-                          f"for the current system: '{hostname}'")
+        raise ConfigError(f'could not find a configuration entry '
+                          f'for the current system: {hostname!r}')
 
     def validate(self):
         site_config = self._pick_config()
@@ -509,8 +534,8 @@ def select_subconfig(self, system_fullname=None,
         # Create local configuration for the current or the requested system
         local_config['systems'] = systems
         for name, section in site_config.items():
-            if name == 'systems':
-                # The systems sections has already been treated
+            if name in ('systems', 'autodetect_methods'):
+                # These sections have already been treated
                 continue
 
             # Convert section to a scoped dict that will handle correctly and
@@ -625,10 +650,14 @@ def find_config_files(config_path=None, config_file=None):
 
 def load_config(*filenames):
     ret = _SiteConfig()
-    getlogger().debug('Loading the generic configuration')
+    getlogger().debug('Loading the builtin configuration')
     ret.update_config(settings.site_configuration, '<builtin>')
     for f in filenames:
-        getlogger().debug(f'Loading configuration file: {filenames!r}')
+        if f == '<builtin>':
+            # The builtin configuration is always loaded at the beginning
+            continue
+
+        getlogger().debug(f'Loading configuration file: {f!r}')
         _, ext = os.path.splitext(f)
         if ext == '.py':
             ret.load_config_python(f)