From 2d240397321a4708cd3ec381f92469872049d27d Mon Sep 17 00:00:00 2001
From: mand35 <mandes.schoenherr@nesi.org.nz>
Date: Tue, 19 Jun 2018 10:40:32 +1200
Subject: [PATCH] save all performance values

One can create multiple Objectives for a singe test case (see example below). This commit improves the logging of the values. It add the tag to the log and it logs all values. (before it was all up to reference is not reached). Thus we can keep better track of all the values observed during the tests.

Reference example:
       p_names = {'creation', 'stat', 'removal'}
        for p_name in p_names:
           self.perf_patterns[p_name] = sn.extractsingle(r'^\s+File '+p_name+
                    '\s+:\s+(?P<'+p_name+'>\S+) ', self.stdout, p_name, float)

        self.reference = {
            'kupe:compute' : {
                'creation' : (7747,  -(2*223.1)/7747, None),
                'stat' :     (16527, -(2*558.2)/16527,None),
                'removal' :  (7355, -(2*173.1)/7355, None)
            },
        }
---
 reframe/core/pipeline.py | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/reframe/core/pipeline.py b/reframe/core/pipeline.py
index 89752ac912..82ba56287a 100644
--- a/reframe/core/pipeline.py
+++ b/reframe/core/pipeline.py
@@ -1019,21 +1019,27 @@ def check_performance(self):
             return
 
         with os_ext.change_dir(self._stagedir):
+            # first check and print all values with references
+            value = {}
             for tag, expr in self.perf_patterns.items():
-                value = evaluate(expr)
+                value[tag] = evaluate(expr)
                 key = '%s:%s' % (self._current_partition.fullname, tag)
                 try:
-                    ref, low_thres, high_thres = self.reference[key]
                     self._perf_logger.info(
-                        'value: %s, reference: %s' %
-                        (value, self.reference[key])
+                        '%s, value: %s, reference: %s' %
+                        (tag, value[tag], self.reference[key])
                     )
+                    ref, low_thres, high_thres = self.reference[key]
                 except KeyError:
                     raise SanityError(
                         "tag `%s' not resolved in references for `%s'" %
                         (tag, self._current_partition.fullname)
                     )
-                evaluate(assert_reference(value, ref, low_thres, high_thres))
+            for tag, expr in self.perf_patterns.items():
+                key = '%s:%s' % (self._current_partition.fullname, tag)
+                ref, low_thres, high_thres = self.reference[key]
+                evaluate(assert_reference(value[tag], ref, low_thres,
+                         high_thres))
 
     def _copy_to_outputdir(self):
         """Copy checks interesting files to the output directory."""