From c120cd1a17f1d110e467213115596d581cd08da0 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Wed, 24 Feb 2021 22:05:23 +0100 Subject: [PATCH 1/3] Log reference values of all performance variables in a single line This also introduces a new function in the `ScopedDict` that returns all the keys in a specific scope. This is used by the logging mechanism to retrieve and log the correct references for the current system partition. --- reframe/core/logging.py | 7 +++++++ reframe/utility/__init__.py | 18 ++++++++++++++++++ unittests/test_utility.py | 12 ++++++++++++ 3 files changed, 37 insertions(+) diff --git a/reframe/core/logging.py b/reframe/core/logging.py index bf68067157..ec1273602a 100644 --- a/reframe/core/logging.py +++ b/reframe/core/logging.py @@ -484,8 +484,15 @@ def _update_check_extras(self): self.extra['check_system'] = self.check.current_system.name if self.check.current_partition: + cp = self.check.current_partition.fullname self.extra['check_partition'] = self.check.current_partition.name + # When logging performance references, we need only those of the + # current system + self.extra['check_reference'] = jsonext.dumps( + self.check.reference.scope(cp) + ) + if self.check.current_environ: self.extra['check_environ'] = self.check.current_environ.name diff --git a/reframe/utility/__init__.py b/reframe/utility/__init__.py index 62c5352a4e..828f5480d4 100644 --- a/reframe/utility/__init__.py +++ b/reframe/utility/__init__.py @@ -727,6 +727,24 @@ def update(self, other): for k, v in scope_dict.items(): self.data[scope][k] = v + def scope(self, scope): + ret = {} + while scope is not None: + if scope in self.data: + for k, v in self.data[scope].items(): + if k not in ret: + ret[k] = v + + if scope == self._global_scope: + scope = None + else: + scope = self._parent_scope(scope) + + if not ret: + raise KeyError(str(scope)) + + return ret + def __str__(self): # just return the internal dictionary return str(self.data) diff --git a/unittests/test_utility.py b/unittests/test_utility.py index 86388d1d3f..e06082f026 100644 --- a/unittests/test_utility.py +++ b/unittests/test_utility.py @@ -952,6 +952,18 @@ def test_scoped_dict_key_resolution(): with pytest.raises(KeyError): scoped_dict[''] + # Scopes must be requested with scope() + assert scoped_dict.scope('a') == {'k1': 1, 'k2': 2, 'k3': 9, 'k4': 10} + assert scoped_dict.scope('a:b') == {'k1': 3, 'k2': 2, 'k3': 4, 'k4': 10} + assert scoped_dict.scope('a:b:c') == {'k1': 3, 'k2': 5, 'k3': 6, 'k4': 10} + assert scoped_dict.scope('*') == {'k1': 7, 'k3': 9, 'k4': 10} + + # This is resolved in scope 'a' + assert scoped_dict.scope('a:z') == {'k1': 1, 'k2': 2, 'k3': 9, 'k4': 10} + + with pytest.raises(KeyError): + scoped_dict.scope(None) + def test_scoped_dict_setitem(): scoped_dict = reframe.utility.ScopedDict({ From 54b5447d79c0a80a5c68d850ca89356d92657e44 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Wed, 24 Feb 2021 22:47:13 +0100 Subject: [PATCH 2/3] Fix unit tests --- reframe/utility/__init__.py | 16 +++++++--------- unittests/test_utility.py | 4 +--- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/reframe/utility/__init__.py b/reframe/utility/__init__.py index 828f5480d4..3aa718c7d5 100644 --- a/reframe/utility/__init__.py +++ b/reframe/utility/__init__.py @@ -729,19 +729,17 @@ def update(self, other): def scope(self, scope): ret = {} - while scope is not None: - if scope in self.data: - for k, v in self.data[scope].items(): + curr_scope = scope + while curr_scope is not None: + if curr_scope in self.data: + for k, v in self.data[curr_scope].items(): if k not in ret: ret[k] = v - if scope == self._global_scope: - scope = None + if curr_scope == self._global_scope: + curr_scope = None else: - scope = self._parent_scope(scope) - - if not ret: - raise KeyError(str(scope)) + curr_scope = self._parent_scope(curr_scope) return ret diff --git a/unittests/test_utility.py b/unittests/test_utility.py index e06082f026..2350ad1762 100644 --- a/unittests/test_utility.py +++ b/unittests/test_utility.py @@ -960,9 +960,7 @@ def test_scoped_dict_key_resolution(): # This is resolved in scope 'a' assert scoped_dict.scope('a:z') == {'k1': 1, 'k2': 2, 'k3': 9, 'k4': 10} - - with pytest.raises(KeyError): - scoped_dict.scope(None) + assert scoped_dict.scope(None) == {} def test_scoped_dict_setitem(): From 677d5c439bf8f02adab8dc0ba3a60f08eb2aad0d Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Thu, 25 Feb 2021 12:07:27 +0100 Subject: [PATCH 3/3] Update documentation --- reframe/utility/__init__.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/reframe/utility/__init__.py b/reframe/utility/__init__.py index 3aa718c7d5..df8eb20d9c 100644 --- a/reframe/utility/__init__.py +++ b/reframe/utility/__init__.py @@ -672,6 +672,7 @@ class ScopedDict(UserDict): ``d['*:k1']`` are all equivalent. If you try to retrieve a whole scope, e.g., ``d['a:b']``, :class:`KeyError` will be raised. + For retrieving scopes, you should use the :func:`scope` function. Key deletion follows the same resolution mechanism as key retrieval, except that you are allowed to delete whole scopes. For example, ``del @@ -727,9 +728,16 @@ def update(self, other): for k, v in scope_dict.items(): self.data[scope][k] = v - def scope(self, scope): + def scope(self, name): + '''Retrieve a whole scope. + + :arg scope: The name of the scope to retrieve. + :returns: A dictionary with the keys that are within the requested + scope. + ''' + ret = {} - curr_scope = scope + curr_scope = name while curr_scope is not None: if curr_scope in self.data: for k, v in self.data[curr_scope].items():