From c27f337d8a7daf14ff95720c3172bc861aa1dd00 Mon Sep 17 00:00:00 2001 From: Eirini Koutsaniti Date: Wed, 18 Aug 2021 10:54:36 +0200 Subject: [PATCH 1/6] Ignore non valid json processor and device topology files --- reframe/frontend/autodetect.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/reframe/frontend/autodetect.py b/reframe/frontend/autodetect.py index 4a1d6d55b8..4331855100 100644 --- a/reframe/frontend/autodetect.py +++ b/reframe/frontend/autodetect.py @@ -183,17 +183,27 @@ def detect_topology(): getlogger().debug( f'> found topology file {topo_file!r}; loading...' ) - part.processor._info = _load_info( - topo_file, _subschema('#/defs/processor_info') + try: + part.processor._info = _load_info( + topo_file, _subschema('#/defs/processor_info') + ) + found_procinfo = True + except json.decoder.JSONDecodeError: + getlogger().debug( + f'> could not load {topo_file!r}; ignoring...' ) - found_procinfo = True if not found_devinfo and os.path.exists(dev_file): getlogger().debug( f'> found devices file {dev_file!r}; loading...' ) - part._devices = _load_info(dev_file, _subschema('#/defs/devices')) - found_devinfo = True + try: + part._devices = _load_info(dev_file, _subschema('#/defs/devices')) + found_devinfo = True + except json.decoder.JSONDecodeError: + getlogger().debug( + f'> could not load {dev_file!r}; ignoring...' + ) if found_procinfo and found_devinfo: continue From e8b9f566beebe71bdbe26d0e1932a5a38d8d655e Mon Sep 17 00:00:00 2001 From: Eirini Koutsaniti Date: Wed, 18 Aug 2021 11:29:29 +0200 Subject: [PATCH 2/6] Split long line --- reframe/frontend/autodetect.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/reframe/frontend/autodetect.py b/reframe/frontend/autodetect.py index 4331855100..1be9b013ce 100644 --- a/reframe/frontend/autodetect.py +++ b/reframe/frontend/autodetect.py @@ -198,7 +198,9 @@ def detect_topology(): f'> found devices file {dev_file!r}; loading...' ) try: - part._devices = _load_info(dev_file, _subschema('#/defs/devices')) + part._devices = _load_info( + dev_file, _subschema('#/defs/devices') + ) found_devinfo = True except json.decoder.JSONDecodeError: getlogger().debug( From 2427a3e913c2674aed6bcad001a82d3aebb459f8 Mon Sep 17 00:00:00 2001 From: Eirini Koutsaniti Date: Wed, 18 Aug 2021 11:31:36 +0200 Subject: [PATCH 3/6] Fix indentation --- reframe/frontend/autodetect.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/reframe/frontend/autodetect.py b/reframe/frontend/autodetect.py index 1be9b013ce..e6b8ef0cb0 100644 --- a/reframe/frontend/autodetect.py +++ b/reframe/frontend/autodetect.py @@ -190,8 +190,8 @@ def detect_topology(): found_procinfo = True except json.decoder.JSONDecodeError: getlogger().debug( - f'> could not load {topo_file!r}; ignoring...' - ) + f'> could not load {topo_file!r}; ignoring...' + ) if not found_devinfo and os.path.exists(dev_file): getlogger().debug( @@ -204,8 +204,8 @@ def detect_topology(): found_devinfo = True except json.decoder.JSONDecodeError: getlogger().debug( - f'> could not load {dev_file!r}; ignoring...' - ) + f'> could not load {dev_file!r}; ignoring...' + ) if found_procinfo and found_devinfo: continue From bab6d57417a4e314270c1e82c62903a21130fab1 Mon Sep 17 00:00:00 2001 From: Eirini Koutsaniti Date: Thu, 19 Aug 2021 09:28:43 +0200 Subject: [PATCH 4/6] Log loading file error --- reframe/frontend/autodetect.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/reframe/frontend/autodetect.py b/reframe/frontend/autodetect.py index e6b8ef0cb0..d9fc8c150a 100644 --- a/reframe/frontend/autodetect.py +++ b/reframe/frontend/autodetect.py @@ -188,7 +188,10 @@ def detect_topology(): topo_file, _subschema('#/defs/processor_info') ) found_procinfo = True - except json.decoder.JSONDecodeError: + except json.decoder.JSONDecodeError as e: + getlogger().debug( + f'> error while loading the file: {e}' + ) getlogger().debug( f'> could not load {topo_file!r}; ignoring...' ) @@ -202,7 +205,10 @@ def detect_topology(): dev_file, _subschema('#/defs/devices') ) found_devinfo = True - except json.decoder.JSONDecodeError: + except json.decoder.JSONDecodeError as e: + getlogger().debug( + f'> error while loading the file: {e}' + ) getlogger().debug( f'> could not load {dev_file!r}; ignoring...' ) From de4dcf49c4e119df6a7558a7a3a3796f41e7ba10 Mon Sep 17 00:00:00 2001 From: Eirini Koutsaniti Date: Thu, 19 Aug 2021 10:24:24 +0200 Subject: [PATCH 5/6] Add unittest for invalid syntax --- unittests/test_autodetect.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/unittests/test_autodetect.py b/unittests/test_autodetect.py index 9d21f5c504..a22889977e 100644 --- a/unittests/test_autodetect.py +++ b/unittests/test_autodetect.py @@ -33,8 +33,32 @@ def exec_ctx(make_exec_ctx_g, tmp_path, monkeypatch): yield from make_exec_ctx_g() +@pytest.fixture +def invalid_topology_exec_ctx(make_exec_ctx_g, tmp_path, monkeypatch): + # Monkey-patch HOME, since topology is always written there + monkeypatch.setenv('HOME', str(tmp_path)) + + # Create invalid processor and devices files + meta_prefix = tmp_path / '.reframe' / 'topology' / 'generic-default' + os.makedirs(meta_prefix) + with open(meta_prefix / 'processor.json', 'w') as fp: + fp.write('{') + + with open(meta_prefix / 'devices.json', 'w') as fp: + fp.write('{') + + yield from make_exec_ctx_g() + + def test_autotect(exec_ctx): detect_topology() part = runtime().system.partitions[0] assert part.processor.info == cpuinfo() assert part.devices == [{'type': 'gpu', 'arch': 'a100', 'num_devices': 8}] + + +def test_autotect_with_invalid_files(invalid_topology_exec_ctx): + detect_topology() + part = runtime().system.partitions[0] + assert part.processor.info == cpuinfo() + assert part.devices == [] From 337801c6562ddc99f4b404a26072e42f8635c82d Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Mon, 23 Aug 2021 11:34:22 +0200 Subject: [PATCH 6/6] Minor style fixes --- reframe/frontend/autodetect.py | 10 ++-------- unittests/test_autodetect.py | 4 ++-- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/reframe/frontend/autodetect.py b/reframe/frontend/autodetect.py index d9fc8c150a..ea037be5db 100644 --- a/reframe/frontend/autodetect.py +++ b/reframe/frontend/autodetect.py @@ -190,10 +190,7 @@ def detect_topology(): found_procinfo = True except json.decoder.JSONDecodeError as e: getlogger().debug( - f'> error while loading the file: {e}' - ) - getlogger().debug( - f'> could not load {topo_file!r}; ignoring...' + f'> could not load {topo_file!r}: {e}: ignoring...' ) if not found_devinfo and os.path.exists(dev_file): @@ -207,10 +204,7 @@ def detect_topology(): found_devinfo = True except json.decoder.JSONDecodeError as e: getlogger().debug( - f'> error while loading the file: {e}' - ) - getlogger().debug( - f'> could not load {dev_file!r}; ignoring...' + f'> could not load {dev_file!r}: {e}: ignoring...' ) if found_procinfo and found_devinfo: diff --git a/unittests/test_autodetect.py b/unittests/test_autodetect.py index a22889977e..4e33f3278c 100644 --- a/unittests/test_autodetect.py +++ b/unittests/test_autodetect.py @@ -34,7 +34,7 @@ def exec_ctx(make_exec_ctx_g, tmp_path, monkeypatch): @pytest.fixture -def invalid_topology_exec_ctx(make_exec_ctx_g, tmp_path, monkeypatch): +def invalid_topo_exec_ctx(make_exec_ctx_g, tmp_path, monkeypatch): # Monkey-patch HOME, since topology is always written there monkeypatch.setenv('HOME', str(tmp_path)) @@ -57,7 +57,7 @@ def test_autotect(exec_ctx): assert part.devices == [{'type': 'gpu', 'arch': 'a100', 'num_devices': 8}] -def test_autotect_with_invalid_files(invalid_topology_exec_ctx): +def test_autotect_with_invalid_files(invalid_topo_exec_ctx): detect_topology() part = runtime().system.partitions[0] assert part.processor.info == cpuinfo()