diff --git a/reframe/core/config.py b/reframe/core/config.py index 55e0dfe3a2..138284a701 100644 --- a/reframe/core/config.py +++ b/reframe/core/config.py @@ -4,6 +4,7 @@ import reframe.core.debug as debug import reframe.core.fields as fields import reframe.utility as util +import reframe.utility.os_ext as os_ext import reframe.utility.typecheck as types from reframe.core.exceptions import (ConfigError, ReframeError, @@ -151,26 +152,26 @@ def create_env(system, partition, name): # Expand variables if sys_prefix: - sys_prefix = os.path.expandvars(sys_prefix) + sys_prefix = os_ext.expandvars(sys_prefix) if sys_stagedir: - sys_stagedir = os.path.expandvars(sys_stagedir) + sys_stagedir = os_ext.expandvars(sys_stagedir) if sys_outputdir: - sys_outputdir = os.path.expandvars(sys_outputdir) + sys_outputdir = os_ext.expandvars(sys_outputdir) if sys_logdir: user_deprecation_warning( "`logdir' attribute in system config is deprecated; " "please use `perflogdir' instead" ) - sys_perflogdir = os.path.expandvars(sys_logdir) + sys_perflogdir = os_ext.expandvars(sys_logdir) if sys_perflogdir: - sys_perflogdir = os.path.expandvars(sys_perflogdir) + sys_perflogdir = os_ext.expandvars(sys_perflogdir) if sys_resourcesdir: - sys_resourcesdir = os.path.expandvars(sys_resourcesdir) + sys_resourcesdir = os_ext.expandvars(sys_resourcesdir) system = System(name=sys_name, descr=sys_descr, diff --git a/reframe/core/environments.py b/reframe/core/environments.py index e8c6f33587..40f8c696cb 100644 --- a/reframe/core/environments.py +++ b/reframe/core/environments.py @@ -64,7 +64,7 @@ def is_loaded(self): """ is_module_loaded = runtime().modules_system.is_module_loaded return (all(map(is_module_loaded, self._modules)) and - all(os.environ.get(k, None) == os.path.expandvars(v) + all(os.environ.get(k, None) == os_ext.expandvars(v) for k, v in self._variables.items())) def load(self): @@ -85,7 +85,7 @@ def load(self): if k in os.environ: self._saved_variables[k] = os.environ[k] - os.environ[k] = os.path.expandvars(v) + os.environ[k] = os_ext.expandvars(v) self._loaded = True diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index a75b8e6cc8..4e8b4171cb 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -306,15 +306,15 @@ def main(): # Adjust system directories if options.prefix: # if prefix is set, reset all other directories - rt.resources.prefix = os.path.expandvars(options.prefix) + rt.resources.prefix = os_ext.expandvars(options.prefix) rt.resources.outputdir = None rt.resources.stagedir = None if options.output: - rt.resources.outputdir = os.path.expandvars(options.output) + rt.resources.outputdir = os_ext.expandvars(options.output) if options.stage: - rt.resources.stagedir = os.path.expandvars(options.stage) + rt.resources.stagedir = os_ext.expandvars(options.stage) if (os_ext.samefile(rt.resources.stage_prefix, rt.resources.output_prefix) and @@ -331,7 +331,7 @@ def main(): # NOTE: we need resources to be configured in order to set the global # perf. logging prefix correctly if options.perflogdir: - rt.resources.perflogdir = os.path.expandvars(options.perflogdir) + rt.resources.perflogdir = os_ext.expandvars(options.perflogdir) logging.LOG_CONFIG_OPTS['handlers.filelog.prefix'] = (rt.resources. perflog_prefix) @@ -369,7 +369,7 @@ def main(): if options.checkpath: load_path = [] for d in options.checkpath: - d = os.path.expandvars(d) + d = os_ext.expandvars(d) if not os.path.exists(d): printer.warning("%s: path `%s' does not exist. Skipping..." % (argparser.prog, d)) diff --git a/reframe/utility/os_ext.py b/reframe/utility/os_ext.py index 05bd9377dd..92d12bad3e 100644 --- a/reframe/utility/os_ext.py +++ b/reframe/utility/os_ext.py @@ -305,3 +305,25 @@ def git_repo_exists(url, timeout=5): return False else: return True + + +def expandvars(path): + """Expand environment variables in ``path`` and + perform any command substitution + + This function is the same as ``os.path.expandvars()``, except that it + understands also the syntax: $(cmd)`` or `cmd`. + """ + cmd_subst = re.compile(r'`(.*)`|\$\((.*)\)') + cmd_subst_m = cmd_subst.search(path) + if not cmd_subst_m: + return os.path.expandvars(path) + + cmd = cmd_subst_m.groups()[0] or cmd_subst_m.groups()[1] + + # We need shell=True to support nested expansion + completed = run_command(cmd, check=True, shell=True) + + # Prepare stdout for inline use + stdout = completed.stdout.replace('\n', ' ').strip() + return cmd_subst.sub(stdout, path) diff --git a/unittests/test_utility.py b/unittests/test_utility.py index e81853d9da..4cc794439c 100644 --- a/unittests/test_utility.py +++ b/unittests/test_utility.py @@ -187,6 +187,50 @@ def test_force_remove_file(self): # Try to remove a non-existent file os_ext.force_remove_file(fp.name) + def test_expandvars_dollar(self): + text = 'Hello, $(echo World)' + self.assertEqual('Hello, World', os_ext.expandvars(text)) + + # Test nested expansion + text = '$(echo Hello, $(echo World))' + self.assertEqual('Hello, World', os_ext.expandvars(text)) + + def test_expandvars_backticks(self): + text = 'Hello, `echo World`' + self.assertEqual('Hello, World', os_ext.expandvars(text)) + + # Test nested expansion + text = '`echo Hello, `echo World``' + self.assertEqual('Hello, World', os_ext.expandvars(text)) + + def test_expandvars_mixed_syntax(self): + text = '`echo Hello, $(echo World)`' + self.assertEqual('Hello, World', os_ext.expandvars(text)) + + text = '$(echo Hello, `echo World`)' + self.assertEqual('Hello, World', os_ext.expandvars(text)) + + def test_expandvars_error(self): + text = 'Hello, $(foo)' + with self.assertRaises(SpawnedProcessError): + os_ext.expandvars(text) + + def test_strange_syntax(self): + text = 'Hello, $(foo`' + self.assertEqual('Hello, $(foo`', os_ext.expandvars(text)) + + text = 'Hello, `foo)' + self.assertEqual('Hello, `foo)', os_ext.expandvars(text)) + + def test_expandvars_nocmd(self): + os.environ['FOO'] = 'World' + text = 'Hello, $FOO' + self.assertEqual('Hello, World', os_ext.expandvars(text)) + + text = 'Hello, ${FOO}' + self.assertEqual('Hello, World', os_ext.expandvars(text)) + del os.environ['FOO'] + class TestCopyTree(unittest.TestCase): def setUp(self):