diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index 5617d3c085..48b0a33964 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -231,6 +231,9 @@ def main(): misc_options.add_argument( '--nocolor', action='store_false', dest='colorize', default=True, help='Disable coloring of output') + misc_options.add_argument( + '--failure-stats', action='store_true', + help='Print failure statistics') misc_options.add_argument('--performance-report', action='store_true', help='Print the performance report') misc_options.add_argument( @@ -630,6 +633,8 @@ def main(): if runner.stats.failures(): printer.info(runner.stats.failure_report()) success = False + if options.failure_stats: + printer.info(runner.stats.failure_stats()) if options.performance_report: printer.info(runner.stats.performance_report()) diff --git a/reframe/frontend/statistics.py b/reframe/frontend/statistics.py index 0ec311ce15..1a85d5b072 100644 --- a/reframe/frontend/statistics.py +++ b/reframe/frontend/statistics.py @@ -97,6 +97,8 @@ def failure_report(self): report.append(' * Job type: %s (id=%s)' % (job_type, jobid)) report.append(' * Maintainers: %s' % check.maintainers) report.append(' * Failing phase: %s' % tf.failed_stage) + report.append(" * Rerun with '-n %s -p %s --system %s'" % + (check.name, environ_name, partname)) reason = ' * Reason: ' if tf.exc_info is not None: from reframe.core.exceptions import format_exception @@ -115,6 +117,50 @@ def failure_report(self): report.append(line_width * '-') return '\n'.join(report) + def failure_stats(self): + failures = {} + current_run = rt.runtime().current_run + for tf in (t for t in self.tasks(current_run) if t.failed): + check = tf.check + partition = check.current_partition + partname = partition.fullname if partition else 'None' + environ_name = (check.current_environ.name + if check.current_environ else 'None') + f = f'[{check.name}, {environ_name}, {partname}]' + if tf.failed_stage not in failures: + failures[tf.failed_stage] = [] + + failures[tf.failed_stage].append(f) + + line_width = 78 + stats_start = line_width * '=' + stats_title = 'FAILURE STATISTICS' + stats_end = line_width * '-' + stats_body = [] + row_format = "{:<11} {:<5} {}" + stats_hline = row_format.format(11*'-', 5*'-', 60*'-') + stats_header = row_format.format('Phase', '#', 'Failing test cases') + num_tests = len(self.tasks(current_run)) + num_failures = 0 + for l in failures.values(): + num_failures += len(l) + + stats_body = [''] + stats_body.append('Total number of test cases: %s' % num_tests) + stats_body.append('Total number of failures: %s' % num_failures) + stats_body.append('') + stats_body.append(stats_header) + stats_body.append(stats_hline) + for p, l in failures.items(): + stats_body.append(row_format.format(p, len(l), l[0])) + for f in l[1:]: + stats_body.append(row_format.format('', '', str(f))) + + if stats_body: + return '\n'.join([stats_start, stats_title, *stats_body, + stats_end]) + return '' + def performance_report(self): line_width = 78 report_start = line_width * '=' diff --git a/unittests/test_cli.py b/unittests/test_cli.py index 82efeaf287..8bcc6bf850 100644 --- a/unittests/test_cli.py +++ b/unittests/test_cli.py @@ -228,6 +228,17 @@ def test_performance_check_failure(self): self.environs) assert self._perflog_exists('PerformanceFailureCheck') + def test_failure_stats(self): + self.checkpath = ['unittests/resources/checks/frontend_checks.py'] + self.more_options = ['-t', 'SanityFailureCheck', '--failure-stats'] + returncode, stdout, stderr = self._run_reframe() + + assert r'FAILURE STATISTICS' in stdout + assert r'sanity 1 [SanityFailureCheck' in stdout + assert 'Traceback' not in stdout + assert 'Traceback' not in stderr + assert returncode != 0 + def test_performance_report(self): self.checkpath = ['unittests/resources/checks/frontend_checks.py'] self.more_options = ['-t', 'PerformanceFailureCheck',