[feat] Support command substitution in variable expansion

reframe/core/config.py

@@ -4,6 +4,7 @@
 import reframe.core.debug as debug
 import reframe.core.fields as fields
 import reframe.utility as util
+import reframe.utility.os_ext as os_ext
 import reframe.utility.typecheck as types
 from reframe.core.exceptions import (ConfigError,
                                      ReframeError,
@@ -151,26 +152,26 @@ def create_env(system, partition, name):
 
             # Expand variables
             if sys_prefix:
-                sys_prefix = os.path.expandvars(sys_prefix)
+                sys_prefix = os_ext.expandvars(sys_prefix)
 
             if sys_stagedir:
-                sys_stagedir = os.path.expandvars(sys_stagedir)
+                sys_stagedir = os_ext.expandvars(sys_stagedir)
 
             if sys_outputdir:
-                sys_outputdir = os.path.expandvars(sys_outputdir)
+                sys_outputdir = os_ext.expandvars(sys_outputdir)
 
             if sys_logdir:
                 user_deprecation_warning(
                     "`logdir' attribute in system config is deprecated; "
                     "please use `perflogdir' instead"
                 )
-                sys_perflogdir = os.path.expandvars(sys_logdir)
+                sys_perflogdir = os_ext.expandvars(sys_logdir)
 
             if sys_perflogdir:
-                sys_perflogdir = os.path.expandvars(sys_perflogdir)
+                sys_perflogdir = os_ext.expandvars(sys_perflogdir)
 
             if sys_resourcesdir:
-                sys_resourcesdir = os.path.expandvars(sys_resourcesdir)
+                sys_resourcesdir = os_ext.expandvars(sys_resourcesdir)
 
             system = System(name=sys_name,
                             descr=sys_descr,

reframe/core/environments.py

@@ -64,7 +64,7 @@ def is_loaded(self):
         """
         is_module_loaded = runtime().modules_system.is_module_loaded
         return (all(map(is_module_loaded, self._modules)) and
-                all(os.environ.get(k, None) == os.path.expandvars(v)
+                all(os.environ.get(k, None) == os_ext.expandvars(v)
                     for k, v in self._variables.items()))
 
     def load(self):
@@ -85,7 +85,7 @@ def load(self):
             if k in os.environ:
                 self._saved_variables[k] = os.environ[k]
 
-            os.environ[k] = os.path.expandvars(v)
+            os.environ[k] = os_ext.expandvars(v)
 
         self._loaded = True
 

reframe/frontend/cli.py

@@ -306,15 +306,15 @@ def main():
     # Adjust system directories
     if options.prefix:
         # if prefix is set, reset all other directories
-        rt.resources.prefix = os.path.expandvars(options.prefix)
+        rt.resources.prefix = os_ext.expandvars(options.prefix)
         rt.resources.outputdir = None
         rt.resources.stagedir  = None
 
     if options.output:
-        rt.resources.outputdir = os.path.expandvars(options.output)
+        rt.resources.outputdir = os_ext.expandvars(options.output)
 
     if options.stage:
-        rt.resources.stagedir = os.path.expandvars(options.stage)
+        rt.resources.stagedir = os_ext.expandvars(options.stage)
 
     if (os_ext.samefile(rt.resources.stage_prefix,
                         rt.resources.output_prefix) and
@@ -331,7 +331,7 @@ def main():
     # NOTE: we need resources to be configured in order to set the global
     # perf. logging prefix correctly
     if options.perflogdir:
-        rt.resources.perflogdir = os.path.expandvars(options.perflogdir)
+        rt.resources.perflogdir = os_ext.expandvars(options.perflogdir)
 
     logging.LOG_CONFIG_OPTS['handlers.filelog.prefix'] = (rt.resources.
                                                           perflog_prefix)
@@ -369,7 +369,7 @@ def main():
     if options.checkpath:
         load_path = []
         for d in options.checkpath:
-            d = os.path.expandvars(d)
+            d = os_ext.expandvars(d)
             if not os.path.exists(d):
                 printer.warning("%s: path `%s' does not exist. Skipping..." %
                                 (argparser.prog, d))

reframe/utility/os_ext.py

@@ -305,3 +305,25 @@ def git_repo_exists(url, timeout=5):
         return False
     else:
         return True
+
+
+def expandvars(path):
+    """Expand environment variables in ``path`` and
+        perform any command substitution
+
+    This function is the same as ``os.path.expandvars()``, except that it
+    understands also the syntax: $(cmd)`` or `cmd`.
+    """
+    cmd_subst = re.compile(r'`(.*)`|\$\((.*)\)')
+    cmd_subst_m = cmd_subst.search(path)
+    if not cmd_subst_m:
+        return os.path.expandvars(path)
+
+    cmd = cmd_subst_m.groups()[0] or cmd_subst_m.groups()[1]
+
+    # We need shell=True to support nested expansion
+    completed = run_command(cmd, check=True, shell=True)
+
+    # Prepare stdout for inline use
+    stdout = completed.stdout.replace('\n', ' ').strip()
+    return cmd_subst.sub(stdout, path)

unittests/test_utility.py

@@ -187,6 +187,50 @@ def test_force_remove_file(self):
         # Try to remove a non-existent file
         os_ext.force_remove_file(fp.name)
 
+    def test_expandvars_dollar(self):
+        text = 'Hello, $(echo World)'
+        self.assertEqual('Hello, World', os_ext.expandvars(text))
+
+        # Test nested expansion
+        text = '$(echo Hello, $(echo World))'
+        self.assertEqual('Hello, World', os_ext.expandvars(text))
+
+    def test_expandvars_backticks(self):
+        text = 'Hello, `echo World`'
+        self.assertEqual('Hello, World', os_ext.expandvars(text))
+
+        # Test nested expansion
+        text = '`echo Hello, `echo World``'
+        self.assertEqual('Hello, World', os_ext.expandvars(text))
+
+    def test_expandvars_mixed_syntax(self):
+        text = '`echo Hello, $(echo World)`'
+        self.assertEqual('Hello, World', os_ext.expandvars(text))
+
+        text = '$(echo Hello, `echo World`)'
+        self.assertEqual('Hello, World', os_ext.expandvars(text))
+
+    def test_expandvars_error(self):
+        text = 'Hello, $(foo)'
+        with self.assertRaises(SpawnedProcessError):
+            os_ext.expandvars(text)
+
+    def test_strange_syntax(self):
+        text = 'Hello, $(foo`'
+        self.assertEqual('Hello, $(foo`', os_ext.expandvars(text))
+
+        text = 'Hello, `foo)'
+        self.assertEqual('Hello, `foo)', os_ext.expandvars(text))
+
+    def test_expandvars_nocmd(self):
+        os.environ['FOO'] = 'World'
+        text = 'Hello, $FOO'
+        self.assertEqual('Hello, World', os_ext.expandvars(text))
+
+        text = 'Hello, ${FOO}'
+        self.assertEqual('Hello, World', os_ext.expandvars(text))
+        del os.environ['FOO']
+
 
 class TestCopyTree(unittest.TestCase):
     def setUp(self):