From 6d8beac3d34fa75c97960ead53d07b6291212e45 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Mon, 31 Oct 2022 00:08:43 +0100 Subject: [PATCH 1/4] Accept any object as a value in `env_vars` --- reframe/core/environments.py | 16 +++++++++++++--- reframe/core/pipeline.py | 4 ++-- unittests/test_environments.py | 9 +++++++-- unittests/test_pipeline.py | 10 +++++----- 4 files changed, 27 insertions(+), 12 deletions(-) diff --git a/reframe/core/environments.py b/reframe/core/environments.py index 4c663145b6..334396be7d 100644 --- a/reframe/core/environments.py +++ b/reframe/core/environments.py @@ -131,9 +131,19 @@ def __eq__(self, other): if not isinstance(other, type(self)): return NotImplemented - return (self.name == other.name and - set(self.modules) == set(other.modules) and - self.env_vars == other.env_vars) + if (self.name != other.name or + set(self.modules) != set(other.modules)): + return False + + # Env. variables are checked against their string representation + for kv0, kv1 in zip(self.env_vars.items(), + other.env_vars.items()): + k0, v0 = kv0 + k1, v1 = kv1 + if k0 != k1 or str(v0) != str(v1): + return False + + return True def __str__(self): return self.name diff --git a/reframe/core/pipeline.py b/reframe/core/pipeline.py index a034f13d9c..c40e7a89ec 100644 --- a/reframe/core/pipeline.py +++ b/reframe/core/pipeline.py @@ -781,11 +781,11 @@ def pipeline_hooks(cls): #: Environment variables to be set before running this test. #: - #: :type: :class:`Dict[str, str]` + #: :type: :class:`Dict[str, object]` #: :default: ``{}`` #: #: .. versionadded:: 4.0.0 - env_vars = variable(typ.Dict[str, str], value={}, loggable=True) + env_vars = variable(typ.Dict[str, object], value={}, loggable=True) #: Environment variables to be set before running this test. #: diff --git a/unittests/test_environments.py b/unittests/test_environments.py index 8992f5347f..a33f905f42 100644 --- a/unittests/test_environments.py +++ b/unittests/test_environments.py @@ -154,8 +154,13 @@ def test_load_overlapping(base_environ): def test_env_equal(base_environ): - env1 = env.Environment('env1', modules=['foo', 'bar']) - env2 = env.Environment('env1', modules=['bar', 'foo']) + env1 = env.Environment('env1', modules=['foo', 'bar'], + env_vars=[('a', '1')]) + env2 = env.Environment('env1', modules=['bar', 'foo'], + env_vars=[('a', 1)]) + + # Environments variables must be checked against their string + # representation assert env1 == env2 assert env2 == env1 diff --git a/unittests/test_pipeline.py b/unittests/test_pipeline.py index 4e8eafdd32..0d6a8350cc 100644 --- a/unittests/test_pipeline.py +++ b/unittests/test_pipeline.py @@ -160,7 +160,7 @@ def test_eq(): def test_environ_setup(hellotest, local_exec_ctx): # Use test environment for the regression check - hellotest.env_vars = {'_FOO_': '1', '_BAR_': '2'} + hellotest.env_vars = {'_FOO_': 1, '_BAR_': 2} hellotest.setup(*local_exec_ctx) for k in hellotest.env_vars.keys(): assert k not in os.environ @@ -1895,13 +1895,13 @@ def set_defaults(self): def _test_variables_deprecation(): with pytest.warns(ReframeDeprecationWarning): class _X(rfm.RunOnlyRegressionTest): - variables = {'FOO': '1'} + variables = {'FOO': 1} test = _X() print('===') - assert test.env_vars['FOO'] == '1' + assert test.env_vars['FOO'] == 1 with pytest.warns(ReframeDeprecationWarning): - test.variables['BAR'] == '2' + test.variables['BAR'] == 2 - assert test.env_vars['BAR'] == '2' + assert test.env_vars['BAR'] == 2 From 6232aab4696ecd44007532f457f5ba8a2f06e5c6 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Mon, 31 Oct 2022 00:34:11 +0100 Subject: [PATCH 2/4] Update tutorial examples --- reframe/core/pipeline.py | 3 +++ tutorials/advanced/parameterized/stream.py | 2 +- tutorials/basics/stream/stream4.py | 2 +- tutorials/cscs-webinar-2022/tests/stream4.py | 2 +- tutorials/cscs-webinar-2022/tests/stream5.py | 2 +- tutorials/cscs-webinar-2022/tests/stream6.py | 2 +- tutorials/cscs-webinar-2022/tests/stream7.py | 2 +- tutorials/cscs-webinar-2022/tests/stream8.py | 4 ++-- tutorials/cscs-webinar-2022/tests/stream9.py | 5 +++-- 9 files changed, 14 insertions(+), 10 deletions(-) diff --git a/reframe/core/pipeline.py b/reframe/core/pipeline.py index c40e7a89ec..80e8ce2214 100644 --- a/reframe/core/pipeline.py +++ b/reframe/core/pipeline.py @@ -781,6 +781,9 @@ def pipeline_hooks(cls): #: Environment variables to be set before running this test. #: + #: The value of the environment variables can be of any type. ReFrame will + #: invoke :func:`str` on it whenever it needs to emit it in a script. + #: #: :type: :class:`Dict[str, object]` #: :default: ``{}`` #: diff --git a/tutorials/advanced/parameterized/stream.py b/tutorials/advanced/parameterized/stream.py index 1593037905..083539e7ea 100644 --- a/tutorials/advanced/parameterized/stream.py +++ b/tutorials/advanced/parameterized/stream.py @@ -68,7 +68,7 @@ def set_num_threads(self): num_threads = self.cores.get(self.current_partition.fullname, 1) self.num_cpus_per_task = num_threads self.env_vars = { - 'OMP_NUM_THREADS': str(num_threads), + 'OMP_NUM_THREADS': num_threads, 'OMP_PLACES': 'cores' } diff --git a/tutorials/basics/stream/stream4.py b/tutorials/basics/stream/stream4.py index fa68616f20..18988e34ae 100644 --- a/tutorials/basics/stream/stream4.py +++ b/tutorials/basics/stream/stream4.py @@ -57,7 +57,7 @@ def set_num_threads(self): num_threads = self.cores.get(self.current_partition.fullname, 1) self.num_cpus_per_task = num_threads self.env_vars = { - 'OMP_NUM_THREADS': str(num_threads), + 'OMP_NUM_THREADS': num_threads, 'OMP_PLACES': 'cores' } diff --git a/tutorials/cscs-webinar-2022/tests/stream4.py b/tutorials/cscs-webinar-2022/tests/stream4.py index 46b5357148..318521252c 100644 --- a/tutorials/cscs-webinar-2022/tests/stream4.py +++ b/tutorials/cscs-webinar-2022/tests/stream4.py @@ -30,7 +30,7 @@ def setup_omp_env(self): procinfo = self.current_partition.processor self.num_cpus_per_task = procinfo.num_cores self.env_vars = { - 'OMP_NUM_THREADS': str(self.num_cpus_per_task), + 'OMP_NUM_THREADS': self.num_cpus_per_task, 'OMP_PLACES': 'cores' } diff --git a/tutorials/cscs-webinar-2022/tests/stream5.py b/tutorials/cscs-webinar-2022/tests/stream5.py index 151d3d6d18..67aa7205eb 100644 --- a/tutorials/cscs-webinar-2022/tests/stream5.py +++ b/tutorials/cscs-webinar-2022/tests/stream5.py @@ -33,7 +33,7 @@ def setup_omp_env(self): procinfo = self.current_partition.processor self.num_cpus_per_task = procinfo.num_cores self.env_vars = { - 'OMP_NUM_THREADS': str(self.num_cpus_per_task), + 'OMP_NUM_THREADS': self.num_cpus_per_task, 'OMP_PLACES': 'cores' } diff --git a/tutorials/cscs-webinar-2022/tests/stream6.py b/tutorials/cscs-webinar-2022/tests/stream6.py index a16a9e8faf..eaa682b91d 100644 --- a/tutorials/cscs-webinar-2022/tests/stream6.py +++ b/tutorials/cscs-webinar-2022/tests/stream6.py @@ -35,7 +35,7 @@ def setup_omp_env(self): procinfo = self.current_partition.processor self.num_cpus_per_task = procinfo.num_cores self.env_vars = { - 'OMP_NUM_THREADS': str(self.num_cpus_per_task), + 'OMP_NUM_THREADS': self.num_cpus_per_task, 'OMP_PLACES': 'cores' } diff --git a/tutorials/cscs-webinar-2022/tests/stream7.py b/tutorials/cscs-webinar-2022/tests/stream7.py index 8abda0b980..aa045da775 100644 --- a/tutorials/cscs-webinar-2022/tests/stream7.py +++ b/tutorials/cscs-webinar-2022/tests/stream7.py @@ -46,7 +46,7 @@ def setup_omp_env(self): procinfo = self.current_partition.processor self.num_cpus_per_task = procinfo.num_cores self.env_vars = { - 'OMP_NUM_THREADS': str(self.num_cpus_per_task), + 'OMP_NUM_THREADS': self.num_cpus_per_task, 'OMP_PLACES': 'cores' } diff --git a/tutorials/cscs-webinar-2022/tests/stream8.py b/tutorials/cscs-webinar-2022/tests/stream8.py index e8b6f6f91c..d8a66b97be 100644 --- a/tutorials/cscs-webinar-2022/tests/stream8.py +++ b/tutorials/cscs-webinar-2022/tests/stream8.py @@ -46,7 +46,7 @@ def setup_omp_env(self): procinfo = self.current_partition.processor self.num_cpus_per_task = procinfo.num_cores self.env_vars = { - 'OMP_NUM_THREADS': str(self.num_cpus_per_task), + 'OMP_NUM_THREADS': self.num_cpus_per_task, 'OMP_PLACES': 'cores' } @@ -67,7 +67,7 @@ class stream_scale_test(stream_test): @run_before('run') def set_cpus_per_task(self): self.num_cpus_per_task = self.num_threads - self.env_vars['OMP_NUM_THREADS'] = str(self.num_cpus_per_task) + self.env_vars['OMP_NUM_THREADS'] = self.num_cpus_per_task @run_after('setup') def skip_if_too_large(self): diff --git a/tutorials/cscs-webinar-2022/tests/stream9.py b/tutorials/cscs-webinar-2022/tests/stream9.py index 3beb5f1f3c..93672a1801 100644 --- a/tutorials/cscs-webinar-2022/tests/stream9.py +++ b/tutorials/cscs-webinar-2022/tests/stream9.py @@ -47,7 +47,7 @@ def setup_omp_env(self): procinfo = self.current_partition.processor self.num_cpus_per_task = procinfo.num_cores self.env_vars = { - 'OMP_NUM_THREADS': str(self.num_cpus_per_task), + 'OMP_NUM_THREADS': self.num_cpus_per_task, 'OMP_PLACES': 'cores' } @@ -83,4 +83,5 @@ def setup_thread_config(self): @run_before('run') def set_cpus_per_task(self): self.num_cpus_per_task = self.num_threads - self.env_vars['OMP_NUM_THREADS'] = str(self.num_cpus_per_task) + + self.env_vars['OMP_NUM_THREADS'] = self.num_cpus_per_task From 14562f62b42fb8674e65493eb4185b0fcce9389a Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Mon, 14 Nov 2022 23:39:59 +0100 Subject: [PATCH 3/4] Convert environment variables to strings when creating Environments --- reframe/core/environments.py | 10 +++++++++- reframe/core/runtime.py | 3 ++- tutorials/basics/stream/stream4.py | 2 +- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/reframe/core/environments.py b/reframe/core/environments.py index 334396be7d..df4eaad3e8 100644 --- a/reframe/core/environments.py +++ b/reframe/core/environments.py @@ -45,7 +45,15 @@ def __init__(self, name, modules=None, env_vars=None, self._name = name self._modules = normalize_module_list(modules) self._module_names = [m['name'] for m in self._modules] - self._env_vars = collections.OrderedDict(env_vars) + + # Convert values of env_vars to strings before storing + if isinstance(env_vars, dict): + env_vars = env_vars.items() + + self._env_vars = collections.OrderedDict() + for k, v in env_vars: + self._env_vars[k] = str(v) + self._extras = extras or {} self._features = features or [] diff --git a/reframe/core/runtime.py b/reframe/core/runtime.py index 893c0e39c4..63341a51ec 100644 --- a/reframe/core/runtime.py +++ b/reframe/core/runtime.py @@ -382,7 +382,8 @@ def __init__(self, modules=[], variables=[]): self._variables = variables def __enter__(self): - new_env = Environment('_rfm_temp_env', self._modules, self._variables) + new_env = Environment('_rfm_temp_env', self._modules, + self._variables.items()) self._environ_save, _ = loadenv(new_env) return new_env diff --git a/tutorials/basics/stream/stream4.py b/tutorials/basics/stream/stream4.py index 18988e34ae..4d637426b2 100644 --- a/tutorials/basics/stream/stream4.py +++ b/tutorials/basics/stream/stream4.py @@ -18,7 +18,7 @@ class StreamMultiSysTest(rfm.RegressionTest): build_system = 'SingleSource' sourcepath = 'stream.c' env_vars = { - 'OMP_NUM_THREADS': '4', + 'OMP_NUM_THREADS': 4, 'OMP_PLACES': 'cores' } reference = { From 3b82d7ad5ff570c2694665be40438e9d596ca90e Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Mon, 14 Nov 2022 23:46:58 +0100 Subject: [PATCH 4/4] Fix setting `env_vars` from the command line --- reframe/core/pipeline.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/reframe/core/pipeline.py b/reframe/core/pipeline.py index 80e8ce2214..46083eb415 100644 --- a/reframe/core/pipeline.py +++ b/reframe/core/pipeline.py @@ -788,7 +788,11 @@ def pipeline_hooks(cls): #: :default: ``{}`` #: #: .. versionadded:: 4.0.0 - env_vars = variable(typ.Dict[str, object], value={}, loggable=True) + env_vars = variable(typ.Dict[str, str], + typ.Dict[str, object], value={}, loggable=True) + # NOTE: We still keep the original type, just to allow setting this + # variable from the command line, because otherwise, ReFrame will not know + # how to convert a value to an arbitrary object. #: Environment variables to be set before running this test. #: