[feat] Enable easy testing of non-default Cray programming environments

docs/running.rst

@@ -908,7 +908,7 @@ ReFrame supports an additional logging facility for recording performance values
 This is configured by the ``perf_logging_config`` variables, whose syntax is the same as for the ``logging_config``:
 
 .. literalinclude:: ../reframe/settings.py
-  :lines: 80-99
+  :lines: 77-96
   :dedent: 4
 
 Performance logging introduces two new log record handlers, specifically designed for this purpose.
@@ -1247,3 +1247,28 @@ For example, using the following options would run a flexible test on all the no
 
 .. warning::
    Test cases resulting from flexible ReFrame tests may not be run using the asynchronous execution policy, because the nodes satisfying the required criteria will be allocated for the first test case, causing all subsequent ones to fail.
+
+Testing non-default Cray Programming Environments
+-------------------------------------------------
+
+.. versionadded:: 2.20
+
+
+Cray machines provide a set of compilers, scientific software libraries and an MPI implementation that is optimized for the Cray hardware.
+These comprise the Cray Programming Environment (PE).
+All the functionality of the PE is structured around the default versions (or modules) of the libraries.
+If a non-default library or a non-default PE are to be tested, users have to do the following after having loaded all of their Cray modules:
+
+.. code:: bash
+
+   export LD_LIBRARY_PATH=$CRAY_LD_LIBRARY_PATH:$LD_LIBRARY_PATH
+
+
+In order to test a non-default Cray Programming Environment with ReFrame you have to pass the ``--non-default-craype``.
+This will cause ReFrame to export ``LD_LIBRARY_PATH`` as shown above.
+Here is an example that shows how to test a non-default Cray PE with ReFrame:
+
+.. code:: bash
+
+   module load cdt/19.08
+   reframe <options> --non-default-craype  -r

reframe/core/pipeline.py

@@ -571,6 +571,7 @@ class RegressionTest(metaclass=RegressionTestMeta):
     _user_environ = fields.TypedField('_user_environ', Environment, type(None))
     _job = fields.TypedField('_job', Job, type(None))
     _build_job = fields.TypedField('_build_job', Job, type(None))
+    _cdt_environ = fields.TypedField('_cdt_environ', Environment)
 
     def __new__(cls, *args, **kwargs):
         obj = super().__new__(cls)
@@ -598,11 +599,11 @@ def _rfm_init(self, name=None, prefix=None):
         self.valid_prog_environs = []
         self.valid_systems = []
         self.sourcepath = ''
-        self.prebuild_cmd  = []
+        self.prebuild_cmd = []
         self.postbuild_cmd = []
         self.executable = os.path.join('.', self.name)
         self.executable_opts = []
-        self.pre_run  = []
+        self.pre_run = []
         self.post_run = []
         self.keep_files = []
         self.readonly_files = []
@@ -675,6 +676,17 @@ def _rfm_init(self, name=None, prefix=None):
         # Weak reference to the test case associated with this check
         self._case = None
 
+        if rt.runtime().non_default_craype:
+            self._cdt_environ = Environment(
+                name='__rfm_cdt_environ',
+                variables={
+                    'LD_LIBRARY_PATH': '$CRAY_LD_LIBRARY_PATH:$LD_LIBRARY_PATH'
+                }
+            )
+        else:
+            # Just an empty environment
+            self._cdt_environ = Environment('__rfm_cdt_environ')
+
     # Export read-only views to interesting fields
     @property
     def current_environ(self):
@@ -879,6 +891,7 @@ def _setup_environ(self, environ):
 
         self.logger.debug("loading user's environment")
         self._user_environ.load()
+        self._cdt_environ.load()
         environ_save.load()
 
     def _setup_paths(self):
@@ -913,10 +926,10 @@ def _setup_job(self, **job_opts):
 
         if self.local:
             scheduler_type = getscheduler('local')
-            launcher_type  = getlauncher('local')
+            launcher_type = getlauncher('local')
         else:
             scheduler_type = self._current_partition.scheduler
-            launcher_type  = self._current_partition.launcher
+            launcher_type = self._current_partition.launcher
 
         self._job = scheduler_type(
             name='rfm_%s_job' % self.name,
@@ -1049,8 +1062,9 @@ def compile(self):
             *self.build_system.emit_build_commands(self._current_environ),
             *self.postbuild_cmd
         ]
-        environs = [self._current_partition.local_env,
-                    self._current_environ, self._user_environ]
+        environs = [self._current_partition.local_env, self._current_environ,
+                    self._user_environ, self._cdt_environ]
+
         self._build_job = getscheduler('local')(
             name='rfm_%s_build' % self.name,
             launcher=getlauncher('local')(),
@@ -1091,8 +1105,9 @@ def run(self):
         exec_cmd = [self.job.launcher.run_command(self.job),
                     self.executable, *self.executable_opts]
         commands = [*self.pre_run, ' '.join(exec_cmd), *self.post_run]
-        environs = [self._current_partition.local_env,
-                    self._current_environ, self._user_environ]
+        environs = [self._current_partition.local_env, self._current_environ,
+                    self._user_environ, self._cdt_environ]
+
         with os_ext.change_dir(self._stagedir):
             try:
                 self._job.prepare(commands, environs, login=True)
@@ -1275,6 +1290,7 @@ def cleanup(self, remove_files=False, unload_env=True):
 
         if unload_env:
             self.logger.debug("unloading test's environment")
+            self._cdt_environ.unload()
             self._user_environ.unload()
             self._current_environ.unload()
             self._current_partition.local_env.unload()

reframe/core/runtime.py

@@ -176,14 +176,17 @@ class RuntimeContext:
 
     This class essentially groups the current host system and the associated
     resources of the framework on the current system.
+    It also encapsulates other runtime parameters that are relevant to the
+    framework's execution.
 
     There is a single instance of this class globally in the framework.
 
     .. note::
        .. versionadded:: 2.13
+
     """
 
-    def __init__(self, dict_config, sysdescr=None):
+    def __init__(self, dict_config, sysdescr=None, **options):
         self._site_config = config.SiteConfiguration(dict_config)
         if sysdescr is not None:
             sysname, _, partname = sysdescr.partition(':')
@@ -202,6 +205,7 @@ def __init__(self, dict_config, sysdescr=None):
         self._modules_system = ModulesSystem.create(
             self._system.modules_system)
         self._current_run = 0
+        self._non_default_craype = options.get('non_default_craype', False)
 
     def _autodetect_system(self):
         """Auto-detect system."""
@@ -260,6 +264,26 @@ def modules_system(self):
         """
         return self._modules_system
 
+    @property
+    def non_default_craype(self):
+        """True if a non-default Cray PE is tested.
+
+        This will cause ReFrame to set the ``LD_LIBRARY_PATH`` as follows after
+        all modules have been loaded:
+
+        .. code:: shell
+
+            export LD_LIBRARY_PATH=$CRAY_LD_LIBRARY_PATH:$LD_LIBRARY_PATH
+
+
+        This property is set through the ``--non-default-craype`` command-line
+        option.
+
+        :type: :class:`bool` (default: :class:`False`)
+
+        """
+        return self._non_default_craype
+
     def show_config(self):
         """Return a textual representation of the current runtime."""
         return str(self._system)
@@ -269,11 +293,11 @@ def show_config(self):
 _runtime_context = None
 
 
-def init_runtime(dict_config, sysname=None):
+def init_runtime(dict_config, sysname=None, **options):
     global _runtime_context
 
     if _runtime_context is None:
-        _runtime_context = RuntimeContext(dict_config, sysname)
+        _runtime_context = RuntimeContext(dict_config, sysname, **options)
 
 
 def runtime():

reframe/frontend/cli.py

@@ -218,6 +218,12 @@ def main():
         help='Disable coloring of output')
     misc_options.add_argument('--performance-report', action='store_true',
                               help='Print the performance report')
+
+    # FIXME: This should move to env_options as soon as
+    # https://github.com/eth-cscs/reframe/pull/946 is merged
+    misc_options.add_argument(
+        '--non-default-craype', action='store_true', default=False,
+        help='Test a non-default Cray PE')
     misc_options.add_argument(
         '--show-config', action='store_true',
         help='Print configuration of the current system and exit')
@@ -270,7 +276,8 @@ def main():
         printer.inc_verbosity(options.verbose)
 
     try:
-        runtime.init_runtime(settings.site_configuration, options.system)
+        runtime.init_runtime(settings.site_configuration, options.system,
+                             non_default_craype=options.non_default_craype)
     except SystemAutodetectionError:
         printer.warning(
             'could not find a configuration entry for the current system; '
@@ -301,7 +308,8 @@ def main():
                 }
             }
         }
-        runtime.init_runtime(settings.site_configuration, 'generic')
+        runtime.init_runtime(settings.site_configuration, 'generic',
+                             non_default_craype=options.non_default_craype)
     except Exception as e:
         printer.error('configuration error: %s' % e)
         printer.verbose(''.join(traceback.format_exception(*sys.exc_info())))