From 94664b3493f5bc877f550ab70e2b7d24c757427e Mon Sep 17 00:00:00 2001 From: Rafael Sarmiento Date: Thu, 4 Apr 2019 10:58:15 +0200 Subject: [PATCH 1/5] add benchmark mode --- reframe/core/pipeline.py | 13 ++++++++++--- reframe/frontend/cli.py | 6 ++++++ reframe/frontend/statistics.py | 16 ++++++++++++++++ 3 files changed, 32 insertions(+), 3 deletions(-) diff --git a/reframe/core/pipeline.py b/reframe/core/pipeline.py index 1ffc2a7e57..a34dc3fe3b 100644 --- a/reframe/core/pipeline.py +++ b/reframe/core/pipeline.py @@ -1110,7 +1110,8 @@ def check_performance(self): # We first evaluate and log all performance values and then we # check them against the reference. This way we always log them # even if the don't meet the reference. - perf_values = [] + self.perf_values = [] + self.perf_keys = [] for tag, expr in self.perf_patterns.items(): value = evaluate(expr) key = '%s:%s' % (self._current_partition.fullname, tag) @@ -1119,11 +1120,17 @@ def check_performance(self): "tag `%s' not resolved in references for `%s'" % (tag, self._current_partition.fullname)) - perf_values.append((value, self.reference[key])) + self.perf_value = value + self.perf_name = key + + self.perf_values.append((value, self.reference[key])) + self.perf_keys.append(key) self._perf_logger.log_performance(logging.INFO, tag, value, *self.reference[key]) - for val, reference in perf_values: + # self.perf_values = perf_values + + for val, reference in self.perf_values: ref, low_thres, high_thres, *_ = reference try: evaluate(assert_reference(val, ref, low_thres, high_thres)) diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index 737df2027b..5770566629 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -231,6 +231,9 @@ def main(): version=reframe.VERSION) misc_options.add_argument('-v', '--verbose', action='count', default=0, help='Increase verbosity level of output') + misc_options.add_argument('--performance-report', action='store_true', + help='Print the performance report') + if len(sys.argv) == 1: argparser.print_help() @@ -546,6 +549,9 @@ def main(): printer.info(runner.stats.failure_report()) success = False + if options.performance_report: + printer.info(runner.stats.performance_report()) + else: printer.info('No action specified. Exiting...') printer.info("Try `%s -h' for a list of available actions." % diff --git a/reframe/frontend/statistics.py b/reframe/frontend/statistics.py index 03c6f7187b..a2740a630b 100644 --- a/reframe/frontend/statistics.py +++ b/reframe/frontend/statistics.py @@ -107,3 +107,19 @@ def failure_report(self): report.append(line_width * '-') return '\n'.join(report) + + def performance_report(self): + current_run = rt.runtime().current_run + line_width = 78 + report = [line_width * '='] + report.append('PERFORMANCE REPORT') + for t in self.tasks(current_run): + report.append(line_width * '-') + report.append('%s' % (t.check.name)) + for k, p in zip(t.check.perf_keys, t.check.perf_values): + system, partition, key = k.split(':') + report.append(' * [%s:%s] %s: %s' % + (system, partition, key, p[0])) + + report.append(line_width * '-') + return '\n'.join(report) From 2179c6a7880ca4b0d60c281de76765ed6e99ac99 Mon Sep 17 00:00:00 2001 From: rafael Date: Thu, 4 Apr 2019 14:52:15 +0200 Subject: [PATCH 2/5] command line option and some fixes --- reframe/core/pipeline.py | 4 ++-- reframe/frontend/cli.py | 1 - reframe/frontend/statistics.py | 16 ++++++++++++---- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/reframe/core/pipeline.py b/reframe/core/pipeline.py index a34dc3fe3b..e8d0334ce0 100644 --- a/reframe/core/pipeline.py +++ b/reframe/core/pipeline.py @@ -566,6 +566,8 @@ def __init__(self, name=None, prefix=None): self.readonly_files = [] self.tags = set() self.maintainers = [] + self.perf_values = [] + self.perf_keys = [] # Strict performance check, if applicable self.strict_check = True @@ -1110,8 +1112,6 @@ def check_performance(self): # We first evaluate and log all performance values and then we # check them against the reference. This way we always log them # even if the don't meet the reference. - self.perf_values = [] - self.perf_keys = [] for tag, expr in self.perf_patterns.items(): value = evaluate(expr) key = '%s:%s' % (self._current_partition.fullname, tag) diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index 5770566629..d29f7efae5 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -234,7 +234,6 @@ def main(): misc_options.add_argument('--performance-report', action='store_true', help='Print the performance report') - if len(sys.argv) == 1: argparser.print_help() sys.exit(1) diff --git a/reframe/frontend/statistics.py b/reframe/frontend/statistics.py index a2740a630b..7b0dbae32f 100644 --- a/reframe/frontend/statistics.py +++ b/reframe/frontend/statistics.py @@ -114,12 +114,20 @@ def performance_report(self): report = [line_width * '='] report.append('PERFORMANCE REPORT') for t in self.tasks(current_run): - report.append(line_width * '-') - report.append('%s' % (t.check.name)) + if t.check.perf_keys: + report.append(line_width * '-') + report.append('%s [%s]' % + (t.check.name, t.check.current_environ)) + for k, p in zip(t.check.perf_keys, t.check.perf_values): system, partition, key = k.split(':') - report.append(' * [%s:%s] %s: %s' % - (system, partition, key, p[0])) + if type(p[1][-1]) is str: + unit = p[1][-1] + else: + unit = '' + + report.append(' * [%s:%s] %s: %s %s' % + (system, partition, key, p[0], unit)) report.append(line_width * '-') return '\n'.join(report) From 8a03e4fd25ee0bcb07d69e8d9295c29e694b9f49 Mon Sep 17 00:00:00 2001 From: rafael Date: Thu, 4 Apr 2019 15:30:02 +0200 Subject: [PATCH 3/5] fix comments --- reframe/core/pipeline.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/reframe/core/pipeline.py b/reframe/core/pipeline.py index e8d0334ce0..f7be3cb6f6 100644 --- a/reframe/core/pipeline.py +++ b/reframe/core/pipeline.py @@ -1120,16 +1120,11 @@ def check_performance(self): "tag `%s' not resolved in references for `%s'" % (tag, self._current_partition.fullname)) - self.perf_value = value - self.perf_name = key - self.perf_values.append((value, self.reference[key])) self.perf_keys.append(key) self._perf_logger.log_performance(logging.INFO, tag, value, *self.reference[key]) - # self.perf_values = perf_values - for val, reference in self.perf_values: ref, low_thres, high_thres, *_ = reference try: From 272b6257d51cf413c6db99f14220b4c66e899b25 Mon Sep 17 00:00:00 2001 From: rafael Date: Tue, 9 Apr 2019 13:05:56 +0200 Subject: [PATCH 4/5] fix comments --- reframe/core/pipeline.py | 13 ++++++++----- reframe/frontend/statistics.py | 29 ++++++++++++++++++----------- 2 files changed, 26 insertions(+), 16 deletions(-) diff --git a/reframe/core/pipeline.py b/reframe/core/pipeline.py index f7be3cb6f6..1de696f797 100644 --- a/reframe/core/pipeline.py +++ b/reframe/core/pipeline.py @@ -566,8 +566,7 @@ def __init__(self, name=None, prefix=None): self.readonly_files = [] self.tags = set() self.maintainers = [] - self.perf_values = [] - self.perf_keys = [] + self._perfvalues = {} # Strict performance check, if applicable self.strict_check = True @@ -660,6 +659,11 @@ def current_system(self): """ return rt.runtime().system + @property + def perfvalues(self): + """The performance values obtained on this test.""" + return util.MappingView(self._perfvalues) + @property def job(self): """The job descriptor associated with this test. @@ -1120,12 +1124,11 @@ def check_performance(self): "tag `%s' not resolved in references for `%s'" % (tag, self._current_partition.fullname)) - self.perf_values.append((value, self.reference[key])) - self.perf_keys.append(key) + self._perfvalues[key] = (value, self.reference[key]) self._perf_logger.log_performance(logging.INFO, tag, value, *self.reference[key]) - for val, reference in self.perf_values: + for val, reference in self._perfvalues.values(): ref, low_thres, high_thres, *_ = reference try: evaluate(assert_reference(val, ref, low_thres, high_thres)) diff --git a/reframe/frontend/statistics.py b/reframe/frontend/statistics.py index 7b0dbae32f..42b8ed47b8 100644 --- a/reframe/frontend/statistics.py +++ b/reframe/frontend/statistics.py @@ -109,25 +109,32 @@ def failure_report(self): return '\n'.join(report) def performance_report(self): - current_run = rt.runtime().current_run line_width = 78 report = [line_width * '='] report.append('PERFORMANCE REPORT') - for t in self.tasks(current_run): - if t.check.perf_keys: - report.append(line_width * '-') - report.append('%s [%s]' % - (t.check.name, t.check.current_environ)) - - for k, p in zip(t.check.perf_keys, t.check.perf_values): - system, partition, key = k.split(':') + previous_name = '' + previous_part = '' + for t in self.tasks(): + if t.check.perfvalues.keys(): + if t.check.name != previous_name: + report.append(line_width * '-') + report.append('%s' % t.check.name) + previous_name = t.check.name + + if t.check.current_partition.fullname != previous_part: + report.append('- %s' % t.check.current_partition.fullname) + previous_part = t.check.current_partition.fullname + + report.append(' - %s' % t.check.current_environ) + + for k, p in t.check.perfvalues.items(): + key = k.split(':')[-1] if type(p[1][-1]) is str: unit = p[1][-1] else: unit = '' - report.append(' * [%s:%s] %s: %s %s' % - (system, partition, key, p[0], unit)) + report.append(' * %s: %s %s' % (key, p[0], unit)) report.append(line_width * '-') return '\n'.join(report) From f072d4f2034fc28ed089c454314c4d103f3863a9 Mon Sep 17 00:00:00 2001 From: rafael Date: Thu, 11 Apr 2019 09:20:34 +0200 Subject: [PATCH 5/5] fix comments --- reframe/core/pipeline.py | 5 ++-- reframe/frontend/cli.py | 46 +++++++++++++++++----------------- reframe/frontend/statistics.py | 17 +++++++------ 3 files changed, 34 insertions(+), 34 deletions(-) diff --git a/reframe/core/pipeline.py b/reframe/core/pipeline.py index 1de696f797..7f75eefdd3 100644 --- a/reframe/core/pipeline.py +++ b/reframe/core/pipeline.py @@ -661,7 +661,6 @@ def current_system(self): @property def perfvalues(self): - """The performance values obtained on this test.""" return util.MappingView(self._perfvalues) @property @@ -1124,11 +1123,11 @@ def check_performance(self): "tag `%s' not resolved in references for `%s'" % (tag, self._current_partition.fullname)) - self._perfvalues[key] = (value, self.reference[key]) + self._perfvalues[key] = (value, *self.reference[key]) self._perf_logger.log_performance(logging.INFO, tag, value, *self.reference[key]) - for val, reference in self._perfvalues.values(): + for val, *reference in self._perfvalues.values(): ref, low_thres, high_thres, *_ = reference try: evaluate(assert_reference(val, ref, low_thres, high_thres)) diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index d29f7efae5..bbcd5f697c 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -188,51 +188,51 @@ def main(): # Miscellaneous options misc_options.add_argument( - '-m', '--module', action='append', default=[], - metavar='MOD', dest='user_modules', - help='Load module MOD before running the regression') + '-C', '--config-file', action='store', dest='config_file', + metavar='FILE', default=os.path.join(reframe.INSTALL_PREFIX, + 'reframe/settings.py'), + help='Specify a custom config-file for the machine. ' + '(default: %s' % os.path.join(reframe.INSTALL_PREFIX, + 'reframe/settings.py')) misc_options.add_argument( '-M', '--map-module', action='append', metavar='MAPPING', dest='module_mappings', default=[], help='Apply a single module mapping') + misc_options.add_argument( + '-m', '--module', action='append', default=[], + metavar='MOD', dest='user_modules', + help='Load module MOD before running the regression') misc_options.add_argument( '--module-mappings', action='store', metavar='FILE', dest='module_map_file', help='Apply module mappings defined in FILE') - misc_options.add_argument( - '--purge-env', action='store_true', dest='purge_env', default=False, - help='Purge modules environment before running any tests') misc_options.add_argument( '--nocolor', action='store_false', dest='colorize', default=True, help='Disable coloring of output') + misc_options.add_argument('--performance-report', action='store_true', + help='Print the performance report') misc_options.add_argument( - '--timestamp', action='store', nargs='?', - const='%FT%T', metavar='TIMEFMT', - help='Append a timestamp component to the regression directories' - '(default format "%%FT%%T")' - ) - misc_options.add_argument( - '--system', action='store', - help='Load SYSTEM configuration explicitly') - misc_options.add_argument( - '-C', '--config-file', action='store', dest='config_file', - metavar='FILE', default=os.path.join(reframe.INSTALL_PREFIX, - 'reframe/settings.py'), - help='Specify a custom config-file for the machine. ' - '(default: %s' % os.path.join(reframe.INSTALL_PREFIX, - 'reframe/settings.py')) + '--purge-env', action='store_true', dest='purge_env', default=False, + help='Purge modules environment before running any tests') misc_options.add_argument( '--show-config', action='store_true', help='Print configuration of the current system and exit') misc_options.add_argument( '--show-config-env', action='store', metavar='ENV', help='Print configuration of environment ENV and exit') + misc_options.add_argument( + '--system', action='store', + help='Load SYSTEM configuration explicitly') + misc_options.add_argument( + '--timestamp', action='store', nargs='?', + const='%FT%T', metavar='TIMEFMT', + help='Append a timestamp component to the regression directories' + '(default format "%%FT%%T")' + ) misc_options.add_argument('-V', '--version', action='version', version=reframe.VERSION) misc_options.add_argument('-v', '--verbose', action='count', default=0, help='Increase verbosity level of output') - misc_options.add_argument('--performance-report', action='store_true', - help='Print the performance report') if len(sys.argv) == 1: argparser.print_help() diff --git a/reframe/frontend/statistics.py b/reframe/frontend/statistics.py index 42b8ed47b8..b4a3294dce 100644 --- a/reframe/frontend/statistics.py +++ b/reframe/frontend/statistics.py @@ -127,14 +127,15 @@ def performance_report(self): report.append(' - %s' % t.check.current_environ) - for k, p in t.check.perfvalues.items(): - key = k.split(':')[-1] - if type(p[1][-1]) is str: - unit = p[1][-1] - else: - unit = '' - - report.append(' * %s: %s %s' % (key, p[0], unit)) + for key, ref in t.check.perfvalues.items(): + var = key.split(':')[-1] + val = ref[0] + try: + unit = ref[4] + except IndexError: + unit = '(no unit specified)' + + report.append(' * %s: %s %s' % (var, val, unit)) report.append(line_width * '-') return '\n'.join(report)