From 34eee316ff7c1f44dd0b78e6fd7997c4aa721d35 Mon Sep 17 00:00:00 2001 From: jgphpc Date: Thu, 10 Sep 2020 15:45:19 +0200 Subject: [PATCH 01/10] Add option to disable JSON report --- config/cscs.py | 3 +- reframe/frontend/cli.py | 57 +++++++++++++++++++++---------------- reframe/schemas/config.json | 2 ++ 3 files changed, 37 insertions(+), 25 deletions(-) diff --git a/config/cscs.py b/config/cscs.py index 256aedf5da..91d9b912f4 100644 --- a/config/cscs.py +++ b/config/cscs.py @@ -1037,7 +1037,8 @@ 'check_search_path': [ 'checks/' ], - 'check_search_recursive': True + 'check_search_recursive': True, + 'save_report_file': True } ] } diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index a7e961dd5a..bae9240cb8 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -423,6 +423,13 @@ def main(): action='store_true', help='Use a login shell for job scripts' ) + argparser.add_argument( + dest='save_report_file', default=True, + envvar='RFM_SAVE_REPORT_FILE', + configvar='general/save_report_file', + action='store_true', + help='Option to output JSON report file' + ) if len(sys.argv) == 1: argparser.print_help() @@ -790,31 +797,33 @@ def print_infoline(param, value): printer.info(runner.stats.performance_report()) # Generate the report for this session - report_file = os.path.normpath( - os_ext.expandvars(rt.get_option('general/0/report_file')) - ) - basedir = os.path.dirname(report_file) - if basedir: - os.makedirs(basedir, exist_ok=True) - - # Build final JSON report - run_stats = runner.stats.json() - session_info.update({ - 'num_cases': run_stats[0]['num_cases'], - 'num_failures': run_stats[-1]['num_failures'] - }) - json_report = { - 'session_info': session_info, - 'runs': run_stats - } - report_file = generate_report_filename(report_file) - try: - with open(report_file, 'w') as fp: - json.dump(json_report, fp, indent=2) - except OSError as e: - printer.warning( - f'failed to generate report in {report_file!r}: {e}' + if site_config.get('general/0/save_report_file'): + report_file = os.path.normpath(os_ext.expandvars( + rt.get_option('general/0/report_file')) ) + basedir = os.path.dirname(report_file) + if basedir: + os.makedirs(basedir, exist_ok=True) + + # Build final JSON report + run_stats = runner.stats.json() + session_info.update({ + 'num_cases': run_stats[0]['num_cases'], + 'num_failures': run_stats[-1]['num_failures'] + }) + json_report = { + 'session_info': session_info, + 'runs': run_stats + } + report_file = generate_report_filename(report_file) + try: + with open(report_file, 'w') as fp: + json.dump(json_report, fp, indent=2) + except OSError as e: + printer.warning( + f'failed to generate report in {report_file!r}: ' + f'{e}' + ) else: printer.error("No action specified. Please specify `-l'/`-L' for " diff --git a/reframe/schemas/config.json b/reframe/schemas/config.json index bd35d64c1b..a75dee1145 100644 --- a/reframe/schemas/config.json +++ b/reframe/schemas/config.json @@ -366,6 +366,7 @@ "purge_environment": {"type": "boolean"}, "report_file": {"type": "string"}, "save_log_files": {"type": "boolean"}, + "save_report_file": {"type": "boolean"}, "target_systems": {"$ref": "#/defs/system_ref"}, "timestamp_dirs": {"type": "string"}, "unload_modules": { @@ -409,6 +410,7 @@ "general/purge_environment": false, "general/report_file": "${HOME}/.reframe/reports/run-report-{sessionid}.json", "general/save_log_files": false, + "general/save_report_file": true, "general/target_systems": ["*"], "general/timestamp_dirs": "", "general/unload_modules": [], From 9707ba8db1d6ab62a957128d60987fe49a849d0d Mon Sep 17 00:00:00 2001 From: jgphpc Date: Fri, 11 Sep 2020 07:51:57 +0200 Subject: [PATCH 02/10] Address PR comments --- reframe/frontend/cli.py | 9 ++++----- reframe/schemas/config.json | 2 +- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index bae9240cb8..bcb71d6950 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -424,10 +424,9 @@ def main(): help='Use a login shell for job scripts' ) argparser.add_argument( - dest='save_report_file', default=True, - envvar='RFM_SAVE_REPORT_FILE', - configvar='general/save_report_file', - action='store_true', + dest='enable_json_report', default=False, + envvar='RFM_enable_json_report', + configvar='general/enable_json_report', help='Option to output JSON report file' ) @@ -797,7 +796,7 @@ def print_infoline(param, value): printer.info(runner.stats.performance_report()) # Generate the report for this session - if site_config.get('general/0/save_report_file'): + if site_config.get('general/0/enable_json_report'): report_file = os.path.normpath(os_ext.expandvars( rt.get_option('general/0/report_file')) ) diff --git a/reframe/schemas/config.json b/reframe/schemas/config.json index a75dee1145..7165fe809c 100644 --- a/reframe/schemas/config.json +++ b/reframe/schemas/config.json @@ -410,7 +410,7 @@ "general/purge_environment": false, "general/report_file": "${HOME}/.reframe/reports/run-report-{sessionid}.json", "general/save_log_files": false, - "general/save_report_file": true, + "general/enable_json_report": false, "general/target_systems": ["*"], "general/timestamp_dirs": "", "general/unload_modules": [], From cf1572eb6a77418045716888dd960846cda4416d Mon Sep 17 00:00:00 2001 From: jgphpc Date: Fri, 11 Sep 2020 10:13:54 +0200 Subject: [PATCH 03/10] Add manual for the enable_json_report option --- docs/manpage.rst | 16 ++++++++++++++++ reframe/frontend/cli.py | 4 ++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/docs/manpage.rst b/docs/manpage.rst index 9e3a62dfac..867ee5ab26 100644 --- a/docs/manpage.rst +++ b/docs/manpage.rst @@ -658,6 +658,22 @@ Here is an alphabetical list of the environment variables recognized by ReFrame: ================================== ================== +.. envvar:: RFM_ENABLE_JSON_REPORT + + Enable json reporting. + + .. table:: + :align: left + + ================================== ================== + Associated command line option N/A + Associated configuration parameter :js:attr:`enable_json_report` general configuration parameter + ================================== ================== + + +.. versionadded:: 3.2 + + .. envvar:: RFM_GRAYLOG_ADDRESS The address of the Graylog server to send performance logs. diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index bcb71d6950..f1e9cb0443 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -424,8 +424,8 @@ def main(): help='Use a login shell for job scripts' ) argparser.add_argument( - dest='enable_json_report', default=False, - envvar='RFM_enable_json_report', + dest='enable_json_report', + envvar='RFM_ENABLE_JSON_REPORT', configvar='general/enable_json_report', help='Option to output JSON report file' ) From c78c6d0b0eae68db049fa94a333878ea70423d5a Mon Sep 17 00:00:00 2001 From: jgphpc Date: Tue, 15 Sep 2020 14:11:24 +0200 Subject: [PATCH 04/10] removing previous name (save_report_file) --- config/cscs.py | 2 +- reframe/schemas/config.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config/cscs.py b/config/cscs.py index d27000b5e3..be80415227 100644 --- a/config/cscs.py +++ b/config/cscs.py @@ -1028,7 +1028,7 @@ 'checks/' ], 'check_search_recursive': True, - 'save_report_file': True + 'enable_json_report': True } ] } diff --git a/reframe/schemas/config.json b/reframe/schemas/config.json index 7165fe809c..f27d67c56e 100644 --- a/reframe/schemas/config.json +++ b/reframe/schemas/config.json @@ -366,7 +366,7 @@ "purge_environment": {"type": "boolean"}, "report_file": {"type": "string"}, "save_log_files": {"type": "boolean"}, - "save_report_file": {"type": "boolean"}, + "enable_json_report": {"type": "boolean"}, "target_systems": {"$ref": "#/defs/system_ref"}, "timestamp_dirs": {"type": "string"}, "unload_modules": { From 382fa75cf80c8d687c1bbabbe244d98760cd4fad Mon Sep 17 00:00:00 2001 From: jgphpc Date: Tue, 15 Sep 2020 16:52:26 +0200 Subject: [PATCH 05/10] fix for review --- config/cscs.py | 1 - docs/manpage.rst | 18 +----------- reframe/frontend/cli.py | 57 ++++++++++++++++--------------------- reframe/schemas/config.json | 4 +-- 4 files changed, 27 insertions(+), 53 deletions(-) diff --git a/config/cscs.py b/config/cscs.py index be80415227..d37ec71e80 100644 --- a/config/cscs.py +++ b/config/cscs.py @@ -1028,7 +1028,6 @@ 'checks/' ], 'check_search_recursive': True, - 'enable_json_report': True } ] } diff --git a/docs/manpage.rst b/docs/manpage.rst index 867ee5ab26..d7c255032f 100644 --- a/docs/manpage.rst +++ b/docs/manpage.rst @@ -241,7 +241,7 @@ Options controlling ReFrame output .. option:: --report-file=FILE The file where ReFrame will store its report. - The ``FILE`` argument may contain the special placeholder ``{sessionid}``, in which case ReFrame will generate a new report each time it is run by appending a counter to the report file. + The ``FILE`` argument may contain the special placeholder ``{sessionid}``, in which case ReFrame will generate a new report each time it is run by appending a counter to the report file. By default, ReFrame will write to $HOME/.reframe/reports/run-report.json. This option can also be set using the :envvar:`RFM_REPORT_FILE` environment variable or the :js:attr:`report_file` general configuration parameter. @@ -658,22 +658,6 @@ Here is an alphabetical list of the environment variables recognized by ReFrame: ================================== ================== -.. envvar:: RFM_ENABLE_JSON_REPORT - - Enable json reporting. - - .. table:: - :align: left - - ================================== ================== - Associated command line option N/A - Associated configuration parameter :js:attr:`enable_json_report` general configuration parameter - ================================== ================== - - -.. versionadded:: 3.2 - - .. envvar:: RFM_GRAYLOG_ADDRESS The address of the Graylog server to send performance logs. diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index f1e9cb0443..adabc67f54 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -423,12 +423,6 @@ def main(): action='store_true', help='Use a login shell for job scripts' ) - argparser.add_argument( - dest='enable_json_report', - envvar='RFM_ENABLE_JSON_REPORT', - configvar='general/enable_json_report', - help='Option to output JSON report file' - ) if len(sys.argv) == 1: argparser.print_help() @@ -796,33 +790,32 @@ def print_infoline(param, value): printer.info(runner.stats.performance_report()) # Generate the report for this session - if site_config.get('general/0/enable_json_report'): - report_file = os.path.normpath(os_ext.expandvars( - rt.get_option('general/0/report_file')) + report_file = os.path.normpath(os_ext.expandvars( + rt.get_option('general/0/report_file')) + ) + basedir = os.path.dirname(report_file) + if basedir: + os.makedirs(basedir, exist_ok=True) + + # Build final JSON report + run_stats = runner.stats.json() + session_info.update({ + 'num_cases': run_stats[0]['num_cases'], + 'num_failures': run_stats[-1]['num_failures'] + }) + json_report = { + 'session_info': session_info, + 'runs': run_stats + } + report_file = generate_report_filename(report_file) + try: + with open(report_file, 'w') as fp: + json.dump(json_report, fp, indent=2) + except OSError as e: + printer.warning( + f'failed to generate report in {report_file!r}: ' + f'{e}' ) - basedir = os.path.dirname(report_file) - if basedir: - os.makedirs(basedir, exist_ok=True) - - # Build final JSON report - run_stats = runner.stats.json() - session_info.update({ - 'num_cases': run_stats[0]['num_cases'], - 'num_failures': run_stats[-1]['num_failures'] - }) - json_report = { - 'session_info': session_info, - 'runs': run_stats - } - report_file = generate_report_filename(report_file) - try: - with open(report_file, 'w') as fp: - json.dump(json_report, fp, indent=2) - except OSError as e: - printer.warning( - f'failed to generate report in {report_file!r}: ' - f'{e}' - ) else: printer.error("No action specified. Please specify `-l'/`-L' for " diff --git a/reframe/schemas/config.json b/reframe/schemas/config.json index f27d67c56e..b914552638 100644 --- a/reframe/schemas/config.json +++ b/reframe/schemas/config.json @@ -366,7 +366,6 @@ "purge_environment": {"type": "boolean"}, "report_file": {"type": "string"}, "save_log_files": {"type": "boolean"}, - "enable_json_report": {"type": "boolean"}, "target_systems": {"$ref": "#/defs/system_ref"}, "timestamp_dirs": {"type": "string"}, "unload_modules": { @@ -408,9 +407,8 @@ "general/module_mappings": [], "general/non_default_craype": false, "general/purge_environment": false, - "general/report_file": "${HOME}/.reframe/reports/run-report-{sessionid}.json", + "general/report_file": "${HOME}/.reframe/reports/run-report.json", "general/save_log_files": false, - "general/enable_json_report": false, "general/target_systems": ["*"], "general/timestamp_dirs": "", "general/unload_modules": [], From 576fb63463faed1599c6c1669ff894712963cf63 Mon Sep 17 00:00:00 2001 From: jgphpc Date: Fri, 18 Sep 2020 15:36:23 +0200 Subject: [PATCH 06/10] fixes for review --- config/cscs.py | 2 +- docs/manpage.rst | 2 +- reframe/frontend/cli.py | 7 +++---- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/config/cscs.py b/config/cscs.py index d37ec71e80..89be772338 100644 --- a/config/cscs.py +++ b/config/cscs.py @@ -1027,7 +1027,7 @@ 'check_search_path': [ 'checks/' ], - 'check_search_recursive': True, + 'check_search_recursive': True } ] } diff --git a/docs/manpage.rst b/docs/manpage.rst index d7c255032f..9e3a62dfac 100644 --- a/docs/manpage.rst +++ b/docs/manpage.rst @@ -241,7 +241,7 @@ Options controlling ReFrame output .. option:: --report-file=FILE The file where ReFrame will store its report. - The ``FILE`` argument may contain the special placeholder ``{sessionid}``, in which case ReFrame will generate a new report each time it is run by appending a counter to the report file. By default, ReFrame will write to $HOME/.reframe/reports/run-report.json. + The ``FILE`` argument may contain the special placeholder ``{sessionid}``, in which case ReFrame will generate a new report each time it is run by appending a counter to the report file. This option can also be set using the :envvar:`RFM_REPORT_FILE` environment variable or the :js:attr:`report_file` general configuration parameter. diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index adabc67f54..a7e961dd5a 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -790,8 +790,8 @@ def print_infoline(param, value): printer.info(runner.stats.performance_report()) # Generate the report for this session - report_file = os.path.normpath(os_ext.expandvars( - rt.get_option('general/0/report_file')) + report_file = os.path.normpath( + os_ext.expandvars(rt.get_option('general/0/report_file')) ) basedir = os.path.dirname(report_file) if basedir: @@ -813,8 +813,7 @@ def print_infoline(param, value): json.dump(json_report, fp, indent=2) except OSError as e: printer.warning( - f'failed to generate report in {report_file!r}: ' - f'{e}' + f'failed to generate report in {report_file!r}: {e}' ) else: From 18a971e2b2fe8cc758eb7f5189ffb4e7baf855c9 Mon Sep 17 00:00:00 2001 From: jgphpc Date: Fri, 18 Sep 2020 15:57:58 +0200 Subject: [PATCH 07/10] Updating doc --- docs/config_reference.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/config_reference.rst b/docs/config_reference.rst index 9b46426d0f..a457086d06 100644 --- a/docs/config_reference.rst +++ b/docs/config_reference.rst @@ -1114,7 +1114,7 @@ General Configuration .. js:attribute:: .general[].report_file :required: No - :default: ``"${HOME}/.reframe/reports/run-report-{sessionid}.json"`` + :default: ``"${HOME}/.reframe/reports/run-report.json"`` The file where ReFrame will store its report. From 9fa707e317135690c658c9e122f3152de97b5187 Mon Sep 17 00:00:00 2001 From: jgphpc Date: Fri, 18 Sep 2020 16:01:24 +0200 Subject: [PATCH 08/10] Updating doc --- docs/manpage.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/manpage.rst b/docs/manpage.rst index 9e3a62dfac..d7c255032f 100644 --- a/docs/manpage.rst +++ b/docs/manpage.rst @@ -241,7 +241,7 @@ Options controlling ReFrame output .. option:: --report-file=FILE The file where ReFrame will store its report. - The ``FILE`` argument may contain the special placeholder ``{sessionid}``, in which case ReFrame will generate a new report each time it is run by appending a counter to the report file. + The ``FILE`` argument may contain the special placeholder ``{sessionid}``, in which case ReFrame will generate a new report each time it is run by appending a counter to the report file. By default, ReFrame will write to $HOME/.reframe/reports/run-report.json. This option can also be set using the :envvar:`RFM_REPORT_FILE` environment variable or the :js:attr:`report_file` general configuration parameter. From c560132e6da3a97ac7c95a643b40716ba813a738 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Fri, 18 Sep 2020 16:20:34 +0200 Subject: [PATCH 09/10] Add a versionchanged annotation in the documentation --- docs/config_reference.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/config_reference.rst b/docs/config_reference.rst index a457086d06..b96f550b69 100644 --- a/docs/config_reference.rst +++ b/docs/config_reference.rst @@ -1119,6 +1119,8 @@ General Configuration The file where ReFrame will store its report. .. versionadded:: 3.1 + .. versionchanged:: 3.2 + Default value has changed to avoid generating a report file per session. .. js:attribute:: .general[].save_log_files From 4331389d4fec922e7c359699998ef1d75f8b0254 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Fri, 18 Sep 2020 16:22:07 +0200 Subject: [PATCH 10/10] Remove mention of default value from man page. --- docs/manpage.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/manpage.rst b/docs/manpage.rst index d7c255032f..9e3a62dfac 100644 --- a/docs/manpage.rst +++ b/docs/manpage.rst @@ -241,7 +241,7 @@ Options controlling ReFrame output .. option:: --report-file=FILE The file where ReFrame will store its report. - The ``FILE`` argument may contain the special placeholder ``{sessionid}``, in which case ReFrame will generate a new report each time it is run by appending a counter to the report file. By default, ReFrame will write to $HOME/.reframe/reports/run-report.json. + The ``FILE`` argument may contain the special placeholder ``{sessionid}``, in which case ReFrame will generate a new report each time it is run by appending a counter to the report file. This option can also be set using the :envvar:`RFM_REPORT_FILE` environment variable or the :js:attr:`report_file` general configuration parameter.