From 330294e9ab385995a8d95705f9bcac9ccbdbf1f1 Mon Sep 17 00:00:00 2001 From: Theofilos Manitaras Date: Fri, 18 Sep 2020 17:04:24 +0200 Subject: [PATCH 1/3] Create `json` utility package for object dumping * Add a custom 'json' utility package which uses the `__rfm_json_encode__` method for json encoding if defined. * Implement the above method for `_Deferred_Expression` objects. * Add unittests for the custom `json` utility package. --- reframe/core/deferrable.py | 12 ++++++++++++ reframe/frontend/cli.py | 5 +++-- reframe/utility/json.py | 26 ++++++++++++++++++++++++++ unittests/test_utility.py | 25 +++++++++++++++++++++++++ 4 files changed, 66 insertions(+), 2 deletions(-) create mode 100644 reframe/utility/json.py diff --git a/reframe/core/deferrable.py b/reframe/core/deferrable.py index 924606b88b..ae9de86d62 100644 --- a/reframe/core/deferrable.py +++ b/reframe/core/deferrable.py @@ -5,6 +5,7 @@ import builtins import functools +import json from reframe.core.exceptions import user_deprecation_warning @@ -77,6 +78,9 @@ def __iter__(self): '''Evaluate the deferred expression and iterate over the result.''' return iter(self.evaluate()) + def __rfm_json_encode__(self): + return self.evaluate() + # Overload Python operators to be able to defer any expression # # NOTE: In the following we are not using `self` for denoting the first @@ -338,3 +342,11 @@ def __abs__(a): @deferrable def __invert__(a): return ~a + + +class DeferredExpressionJSONEncoder(json.JSONEncoder): + def default(self, obj): + if isinstance(obj, _DeferredExpression): + return obj.evaluate() + + return json.JSONEncoder.default(self, obj) diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index a7e961dd5a..113a113705 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -3,8 +3,8 @@ # # SPDX-License-Identifier: BSD-3-Clause -import inspect import json +import inspect import os import re import socket @@ -21,6 +21,7 @@ import reframe.frontend.check_filters as filters import reframe.frontend.dependency as dependency import reframe.utility.os_ext as os_ext +import reframe.utility.json as jsonext from reframe.core.exceptions import ( EnvironError, ConfigError, ReframeError, ReframeDeprecationWarning, ReframeFatalError, @@ -810,7 +811,7 @@ def print_infoline(param, value): report_file = generate_report_filename(report_file) try: with open(report_file, 'w') as fp: - json.dump(json_report, fp, indent=2) + jsonext.dump(json_report, fp, indent=2) except OSError as e: printer.warning( f'failed to generate report in {report_file!r}: {e}' diff --git a/reframe/utility/json.py b/reframe/utility/json.py new file mode 100644 index 0000000000..6c3361ee8e --- /dev/null +++ b/reframe/utility/json.py @@ -0,0 +1,26 @@ +# Copyright 2016-2020 Swiss National Supercomputing Centre (CSCS/ETH Zurich) +# ReFrame Project Developers. See the top-level LICENSE file for details. +# +# SPDX-License-Identifier: BSD-3-Clause + +import json + + +class _ReframeJsonEncoder(json.JSONEncoder): + def default(self, obj): + if hasattr(obj, '__rfm_json_encode__'): + return obj.__rfm_json_encode__() + + return json.JSONEncoder.default(self, obj) + + +def dump(obj, fp, **kwargs): + fn_kwargs = kwargs + kwargs['cls'] = _ReframeJsonEncoder + return json.dump(obj, fp, **kwargs) + + +def dumps(obj, **kwargs): + fn_kwargs = kwargs + kwargs['cls'] = _ReframeJsonEncoder + return json.dumps(obj, **kwargs) diff --git a/unittests/test_utility.py b/unittests/test_utility.py index 4cc1329334..0e014bbced 100644 --- a/unittests/test_utility.py +++ b/unittests/test_utility.py @@ -12,7 +12,9 @@ import reframe import reframe.core.fields as fields import reframe.utility as util +import reframe.utility.json as jsonext import reframe.utility.os_ext as os_ext +import reframe.utility.sanity as sn from reframe.core.exceptions import (SpawnedProcessError, SpawnedProcessTimeout) @@ -1293,3 +1295,26 @@ def test_cray_cle_info_missing_parts(tmp_path): assert cle_info.date is None assert cle_info.network is None assert cle_info.patchset == '09' + + +def test_jsonext_dump(tmp_path): + json_dump = tmp_path / 'test.json' + json_dump_indent = tmp_path / 'test_indent.json' + with open(json_dump, 'w') as f: + jsonext.dump({'foo': sn.defer(['bar'])}, f) + + with open(json_dump, 'r') as f: + assert '{"foo": ["bar"]}' == f.read() + + with open(json_dump_indent, 'w') as f: + jsonext.dump({'foo': sn.defer(['bar'])}, f, indent=2) + + with open(json_dump_indent, 'r') as f: + assert '{\n "foo": [\n "bar"\n ]\n}' == f.read() + + +def test_jsonext_dumps(): + assert '"foo"' == jsonext.dumps('foo') + assert '{"foo": ["bar"]}' == jsonext.dumps({'foo': sn.defer(['bar'])}) + assert ('{\n "foo": [\n "bar"\n ]\n}' == + jsonext.dumps({'foo': sn.defer(['bar'])}, indent=2)) From b321285e0e94c95f9c1178deb9448a640b515a62 Mon Sep 17 00:00:00 2001 From: Theofilos Manitaras Date: Fri, 18 Sep 2020 17:12:17 +0200 Subject: [PATCH 2/3] Remove stale code --- reframe/core/deferrable.py | 8 -------- reframe/frontend/cli.py | 2 +- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/reframe/core/deferrable.py b/reframe/core/deferrable.py index ae9de86d62..7687bf6c16 100644 --- a/reframe/core/deferrable.py +++ b/reframe/core/deferrable.py @@ -342,11 +342,3 @@ def __abs__(a): @deferrable def __invert__(a): return ~a - - -class DeferredExpressionJSONEncoder(json.JSONEncoder): - def default(self, obj): - if isinstance(obj, _DeferredExpression): - return obj.evaluate() - - return json.JSONEncoder.default(self, obj) diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index 113a113705..35c2ea2453 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -3,8 +3,8 @@ # # SPDX-License-Identifier: BSD-3-Clause -import json import inspect +import json import os import re import socket From 577316e2b5a67004605bf180c6d6afe8a030653e Mon Sep 17 00:00:00 2001 From: Theofilos Manitaras Date: Sat, 19 Sep 2020 20:33:03 +0200 Subject: [PATCH 3/3] Address PR comments --- reframe/core/deferrable.py | 1 - reframe/utility/json.py | 2 -- unittests/test_utility.py | 21 ++++++++++----------- 3 files changed, 10 insertions(+), 14 deletions(-) diff --git a/reframe/core/deferrable.py b/reframe/core/deferrable.py index 7687bf6c16..e3274bda64 100644 --- a/reframe/core/deferrable.py +++ b/reframe/core/deferrable.py @@ -5,7 +5,6 @@ import builtins import functools -import json from reframe.core.exceptions import user_deprecation_warning diff --git a/reframe/utility/json.py b/reframe/utility/json.py index 6c3361ee8e..2d7e4675f7 100644 --- a/reframe/utility/json.py +++ b/reframe/utility/json.py @@ -15,12 +15,10 @@ def default(self, obj): def dump(obj, fp, **kwargs): - fn_kwargs = kwargs kwargs['cls'] = _ReframeJsonEncoder return json.dump(obj, fp, **kwargs) def dumps(obj, **kwargs): - fn_kwargs = kwargs kwargs['cls'] = _ReframeJsonEncoder return json.dumps(obj, **kwargs) diff --git a/unittests/test_utility.py b/unittests/test_utility.py index 0e014bbced..8465d3f9f0 100644 --- a/unittests/test_utility.py +++ b/unittests/test_utility.py @@ -1299,22 +1299,21 @@ def test_cray_cle_info_missing_parts(tmp_path): def test_jsonext_dump(tmp_path): json_dump = tmp_path / 'test.json' - json_dump_indent = tmp_path / 'test_indent.json' - with open(json_dump, 'w') as f: - jsonext.dump({'foo': sn.defer(['bar'])}, f) + with open(json_dump, 'w') as fp: + jsonext.dump({'foo': sn.defer(['bar'])}, fp) - with open(json_dump, 'r') as f: - assert '{"foo": ["bar"]}' == f.read() + with open(json_dump, 'r') as fp: + assert '{"foo": ["bar"]}' == fp.read() - with open(json_dump_indent, 'w') as f: - jsonext.dump({'foo': sn.defer(['bar'])}, f, indent=2) + with open(json_dump, 'w') as fp: + jsonext.dump({'foo': sn.defer(['bar'])}, fp, separators=(',', ':')) - with open(json_dump_indent, 'r') as f: - assert '{\n "foo": [\n "bar"\n ]\n}' == f.read() + with open(json_dump, 'r') as fp: + assert '{"foo":["bar"]}' == fp.read() def test_jsonext_dumps(): assert '"foo"' == jsonext.dumps('foo') assert '{"foo": ["bar"]}' == jsonext.dumps({'foo': sn.defer(['bar'])}) - assert ('{\n "foo": [\n "bar"\n ]\n}' == - jsonext.dumps({'foo': sn.defer(['bar'])}, indent=2)) + assert '{"foo":["bar"]}' == jsonext.dumps({'foo': sn.defer(['bar'])}, + separators=(',', ':'))