From 81694180b0ec5b39aa323dce62d553dbc4f206c1 Mon Sep 17 00:00:00 2001 From: Eirini Koutsaniti Date: Tue, 5 Apr 2022 17:21:47 +0200 Subject: [PATCH 1/5] Improve performance reference documentation --- reframe/core/pipeline.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/reframe/core/pipeline.py b/reframe/core/pipeline.py index 2a227b0d14..2db659a426 100644 --- a/reframe/core/pipeline.py +++ b/reframe/core/pipeline.py @@ -609,6 +609,28 @@ def pipeline_hooks(cls): #: } #: } #: + #: Here are some examples to understand how to set the performance tuple, + #: in case the expected reference is a negative or a positive number: + #: + #: +-------------------------------+--------------+---------------------+----------------------+ + #: | **Performance Tuple** | **Expected** | **Lowest accepted** | **Highest accepted** | + #: +-------------------------------+--------------+---------------------+----------------------+ + #: | (100, -0.01, 0.02, 'unit') | 100 unit | 99 unit | 102 unit | + #: +-------------------------------+--------------+---------------------+----------------------+ + #: | (100, -0.01, None, 'unit') | 100 unit | 99 unit | inf unit | + #: +-------------------------------+--------------+---------------------+----------------------+ + #: | (100, None, 0.02, 'unit') | 100 unit | -inf unit | 102 unit | + #: +-------------------------------+--------------+---------------------+----------------------+ + #: | (-100, -0.01, 0.02, 'unit') | -100 unit | -101 unit | -98 unit | + #: +-------------------------------+--------------+---------------------+----------------------+ + #: | (-100, -0.01, None, 'unit') | -100 unit | -101 unit | inf unit | + #: +-------------------------------+--------------+---------------------+----------------------+ + #: | (-100, None, 0.02, 'unit') | -100 unit | -inf unit | -98 unit | + #: +-------------------------------+--------------+---------------------+----------------------+ + #: + #: More information in the `assert_reference function + #: `__. + #: #: :type: A scoped dictionary with system names as scopes or :class:`None` #: :default: ``{}`` #: From 21d65b1bf2c878ed8b897a2125f01d75d00a8e17 Mon Sep 17 00:00:00 2001 From: Eirini Koutsaniti Date: Tue, 5 Apr 2022 17:38:03 +0200 Subject: [PATCH 2/5] fix pep8 issues --- reframe/core/pipeline.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/reframe/core/pipeline.py b/reframe/core/pipeline.py index 2db659a426..36230f2b26 100644 --- a/reframe/core/pipeline.py +++ b/reframe/core/pipeline.py @@ -612,24 +612,24 @@ def pipeline_hooks(cls): #: Here are some examples to understand how to set the performance tuple, #: in case the expected reference is a negative or a positive number: #: - #: +-------------------------------+--------------+---------------------+----------------------+ - #: | **Performance Tuple** | **Expected** | **Lowest accepted** | **Highest accepted** | - #: +-------------------------------+--------------+---------------------+----------------------+ - #: | (100, -0.01, 0.02, 'unit') | 100 unit | 99 unit | 102 unit | - #: +-------------------------------+--------------+---------------------+----------------------+ - #: | (100, -0.01, None, 'unit') | 100 unit | 99 unit | inf unit | - #: +-------------------------------+--------------+---------------------+----------------------+ - #: | (100, None, 0.02, 'unit') | 100 unit | -inf unit | 102 unit | - #: +-------------------------------+--------------+---------------------+----------------------+ - #: | (-100, -0.01, 0.02, 'unit') | -100 unit | -101 unit | -98 unit | - #: +-------------------------------+--------------+---------------------+----------------------+ - #: | (-100, -0.01, None, 'unit') | -100 unit | -101 unit | inf unit | - #: +-------------------------------+--------------+---------------------+----------------------+ - #: | (-100, None, 0.02, 'unit') | -100 unit | -inf unit | -98 unit | - #: +-------------------------------+--------------+---------------------+----------------------+ + #: +-------------------------------+--------------+---------------------+----------------------+ # noqa: E501 + #: | **Performance Tuple** | **Expected** | **Lowest accepted** | **Highest accepted** | # noqa: E501 + #: +-------------------------------+--------------+---------------------+----------------------+ # noqa: E501 + #: | (100, -0.01, 0.02, 'unit') | 100 unit | 99 unit | 102 unit | # noqa: E501 + #: +-------------------------------+--------------+---------------------+----------------------+ # noqa: E501 + #: | (100, -0.01, None, 'unit') | 100 unit | 99 unit | inf unit | # noqa: E501 + #: +-------------------------------+--------------+---------------------+----------------------+ # noqa: E501 + #: | (100, None, 0.02, 'unit') | 100 unit | -inf unit | 102 unit | # noqa: E501 + #: +-------------------------------+--------------+---------------------+----------------------+ # noqa: E501 + #: | (-100, -0.01, 0.02, 'unit') | -100 unit | -101 unit | -98 unit | # noqa: E501 + #: +-------------------------------+--------------+---------------------+----------------------+ # noqa: E501 + #: | (-100, -0.01, None, 'unit') | -100 unit | -101 unit | inf unit | # noqa: E501 + #: +-------------------------------+--------------+---------------------+----------------------+ # noqa: E501 + #: | (-100, None, 0.02, 'unit') | -100 unit | -inf unit | -98 unit | # noqa: E501 + #: +-------------------------------+--------------+---------------------+----------------------+ # noqa: E501 #: #: More information in the `assert_reference function - #: `__. + #: `__. # noqa: E501 #: #: :type: A scoped dictionary with system names as scopes or :class:`None` #: :default: ``{}`` From 37a6c7be5c79607e4e906ed97549a6b499dacf85 Mon Sep 17 00:00:00 2001 From: Eirini Koutsaniti Date: Wed, 6 Apr 2022 10:50:40 +0200 Subject: [PATCH 3/5] Address PR comments --- reframe/core/pipeline.py | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/reframe/core/pipeline.py b/reframe/core/pipeline.py index 36230f2b26..05c5e91916 100644 --- a/reframe/core/pipeline.py +++ b/reframe/core/pipeline.py @@ -612,24 +612,18 @@ def pipeline_hooks(cls): #: Here are some examples to understand how to set the performance tuple, #: in case the expected reference is a negative or a positive number: #: - #: +-------------------------------+--------------+---------------------+----------------------+ # noqa: E501 - #: | **Performance Tuple** | **Expected** | **Lowest accepted** | **Highest accepted** | # noqa: E501 - #: +-------------------------------+--------------+---------------------+----------------------+ # noqa: E501 - #: | (100, -0.01, 0.02, 'unit') | 100 unit | 99 unit | 102 unit | # noqa: E501 - #: +-------------------------------+--------------+---------------------+----------------------+ # noqa: E501 - #: | (100, -0.01, None, 'unit') | 100 unit | 99 unit | inf unit | # noqa: E501 - #: +-------------------------------+--------------+---------------------+----------------------+ # noqa: E501 - #: | (100, None, 0.02, 'unit') | 100 unit | -inf unit | 102 unit | # noqa: E501 - #: +-------------------------------+--------------+---------------------+----------------------+ # noqa: E501 - #: | (-100, -0.01, 0.02, 'unit') | -100 unit | -101 unit | -98 unit | # noqa: E501 - #: +-------------------------------+--------------+---------------------+----------------------+ # noqa: E501 - #: | (-100, -0.01, None, 'unit') | -100 unit | -101 unit | inf unit | # noqa: E501 - #: +-------------------------------+--------------+---------------------+----------------------+ # noqa: E501 - #: | (-100, None, 0.02, 'unit') | -100 unit | -inf unit | -98 unit | # noqa: E501 - #: +-------------------------------+--------------+---------------------+----------------------+ # noqa: E501 + #: =========================== ============ ========== =========== + #: **Performance Tuple** **Expected** **Lowest** **Highest** + #: (100, -0.01, 0.02, 'MB/s') 100 MB/s 99 MB/s 102 MB/s + #: (100, -0.01, None, 'MB/s') 100 MB/s 99 MB/s inf MB/s + #: (100, None, 0.02, 'MB/s') 100 MB/s -inf MB/s 102 MB/s + #: (-100, -0.01, 0.02, 'C') -100 C -101 C -98 C + #: (-100, -0.01, None, 'C') -100 C -101 C inf C + #: (-100, None, 0.02, 'C') -100 C -inf C -98 C + #: =========================== ============ ========== =========== #: #: More information in the `assert_reference function - #: `__. # noqa: E501 + #: `__. #: #: :type: A scoped dictionary with system names as scopes or :class:`None` #: :default: ``{}`` From 1e88d3e1717d662083f792c757148227139ff2c1 Mon Sep 17 00:00:00 2001 From: Eirini Koutsaniti Date: Wed, 6 Apr 2022 14:06:14 +0200 Subject: [PATCH 4/5] Address PR comments --- reframe/core/pipeline.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/reframe/core/pipeline.py b/reframe/core/pipeline.py index 05c5e91916..e57b29ed8b 100644 --- a/reframe/core/pipeline.py +++ b/reframe/core/pipeline.py @@ -622,8 +622,8 @@ def pipeline_hooks(cls): #: (-100, None, 0.02, 'C') -100 C -inf C -98 C #: =========================== ============ ========== =========== #: - #: More information in the `assert_reference function - #: `__. + #: More information in the + #: :func:`~reframe.utility.sanity.assert_reference` function. #: #: :type: A scoped dictionary with system names as scopes or :class:`None` #: :default: ``{}`` From 7b8aa0fa48ccce5c8a3c40c1bf9d369114787781 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Thu, 7 Apr 2022 00:12:50 +0200 Subject: [PATCH 5/5] Doc improvements --- reframe/core/pipeline.py | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/reframe/core/pipeline.py b/reframe/core/pipeline.py index e57b29ed8b..85cc6b92c7 100644 --- a/reframe/core/pipeline.py +++ b/reframe/core/pipeline.py @@ -609,21 +609,24 @@ def pipeline_hooks(cls): #: } #: } #: - #: Here are some examples to understand how to set the performance tuple, - #: in case the expected reference is a negative or a positive number: - #: - #: =========================== ============ ========== =========== - #: **Performance Tuple** **Expected** **Lowest** **Highest** - #: (100, -0.01, 0.02, 'MB/s') 100 MB/s 99 MB/s 102 MB/s - #: (100, -0.01, None, 'MB/s') 100 MB/s 99 MB/s inf MB/s - #: (100, None, 0.02, 'MB/s') 100 MB/s -inf MB/s 102 MB/s - #: (-100, -0.01, 0.02, 'C') -100 C -101 C -98 C - #: (-100, -0.01, None, 'C') -100 C -101 C inf C - #: (-100, None, 0.02, 'C') -100 C -inf C -98 C - #: =========================== ============ ========== =========== - #: - #: More information in the - #: :func:`~reframe.utility.sanity.assert_reference` function. + #: To better understand how to set the performance reference tuple, here + #: are some examples with both positive and negative reference values: + #: + #: ============================== ============ ========== =========== + #: **Performance Tuple** **Expected** **Lowest** **Highest** + #: ``(100, -0.01, 0.02, 'MB/s')`` 100 MB/s 99 MB/s 102 MB/s + #: ``(100, -0.01, None, 'MB/s')`` 100 MB/s 99 MB/s inf MB/s + #: ``(100, None, 0.02, 'MB/s')`` 100 MB/s -inf MB/s 102 MB/s + #: ``(-100, -0.01, 0.02, 'C')`` -100 C -101 C -98 C + #: ``(-100, -0.01, None, 'C')`` -100 C -101 C inf C + #: ``(-100, None, 0.02, 'C')`` -100 C -inf C -98 C + #: ============================== ============ ========== =========== + #: + #: During the performance stage of the pipeline, the reference tuple + #: elements, except the unit, are passed to the + #: :func:`~reframe.utility.sanity.assert_reference` function along with the + #: obtained performance value in order to actually assess whether the test + #: passes the performance check or not. #: #: :type: A scoped dictionary with system names as scopes or :class:`None` #: :default: ``{}``