diff --git a/ci-scripts/ci-runner.bash b/ci-scripts/ci-runner.bash index 805d69f6b4..5cc5d8c33e 100644 --- a/ci-scripts/ci-runner.bash +++ b/ci-scripts/ci-runner.bash @@ -137,13 +137,15 @@ fi if [[ $(hostname) =~ tsa ]]; then # FIXME: Temporary workaround until we have a reframe module on Tsa module load python - python3 -m venv venv.unittests - source venv.unittests/bin/activate - pip install -r requirements.txt else module load reframe fi +# Always install our requirements +python3 -m venv venv.unittests +source venv.unittests/bin/activate +pip install -r requirements.txt + echo "==============" echo "Loaded Modules" echo "==============" diff --git a/reframe/core/config.py b/reframe/core/config.py index b44b632a23..69eb8001ee 100644 --- a/reframe/core/config.py +++ b/reframe/core/config.py @@ -4,14 +4,20 @@ # SPDX-License-Identifier: BSD-3-Clause import collections.abc +import json +import jsonschema +import os import re +import tempfile +import reframe import reframe.core.debug as debug import reframe.core.fields as fields import reframe.utility as util import reframe.utility.os_ext as os_ext import reframe.utility.typecheck as types -from reframe.core.exceptions import (ConfigError, ReframeFatalError) +from reframe.core.exceptions import (ConfigError, ReframeError, + ReframeFatalError) _settings = None @@ -222,3 +228,147 @@ def create_env(system, partition, name): system.add_partition(part) self._systems[sys_name] = system + + +def convert_old_config(filename): + old_config = load_settings_from_file(filename) + converted = { + 'systems': [], + 'environments': [], + 'logging': [], + 'perf_logging': [], + } + old_systems = old_config.site_configuration['systems'].items() + for sys_name, sys_specs in old_systems: + sys_dict = {'name': sys_name} + sys_dict.update(sys_specs) + + # Make variables dictionary into a list of lists + if 'variables' in sys_specs: + sys_dict['variables'] = [ + [vname, v] for vname, v in sys_dict['variables'].items() + ] + + # Make partitions dictionary into a list + if 'partitions' in sys_specs: + sys_dict['partitions'] = [] + for pname, p in sys_specs['partitions'].items(): + new_p = {'name': pname} + new_p.update(p) + if p['scheduler'] == 'nativeslurm': + new_p['scheduler'] = 'slurm' + new_p['launcher'] = 'srun' + elif p['scheduler'] == 'local': + new_p['scheduler'] = 'local' + new_p['launcher'] = 'local' + else: + sched, launch, *_ = p['scheduler'].split('+') + new_p['scheduler'] = sched + new_p['launcher'] = launch + + # Make resources dictionary into a list + if 'resources' in p: + new_p['resources'] = [ + {'name': rname, 'options': r} + for rname, r in p['resources'].items() + ] + + # Make variables dictionary into a list of lists + if 'variables' in p: + new_p['variables'] = [ + [vname, v] for vname, v in p['variables'].items() + ] + + if 'container_platforms' in p: + new_p['container_platforms'] = [] + for cname, c in p['container_platforms'].items(): + new_c = {'name': cname} + new_c.update(c) + if 'variables' in c: + new_c['variables'] = [ + [vn, v] for vn, v in c['variables'].items() + ] + + new_p['container_platforms'].append(new_c) + + sys_dict['partitions'].append(new_p) + + converted['systems'].append(sys_dict) + + old_environs = old_config.site_configuration['environments'].items() + for env_target, env_entries in old_environs: + for ename, e in env_entries.items(): + new_env = {'name': ename} + if env_target != '*': + new_env['target_systems'] = [env_target] + + new_env.update(e) + + # Convert variables dictionary to a list of lists + if 'variables' in e: + new_env['variables'] = [ + [vname, v] for vname, v in e['variables'].items() + ] + + # Type attribute is not used anymore + if 'type' in new_env: + del new_env['type'] + + converted['environments'].append(new_env) + + if 'modes' in old_config.site_configuration: + converted['modes'] = [] + old_modes = old_config.site_configuration['modes'].items() + for target_mode, mode_entries in old_modes: + for mname, m in mode_entries.items(): + new_mode = {'name': mname, 'options': m} + if target_mode != '*': + new_mode['target_systems'] = [target_mode] + + converted['modes'].append(new_mode) + + def update_logging_config(log_name, original_log): + new_handlers = [] + for h in original_log['handlers']: + new_h = h + new_h['level'] = h['level'].lower() + new_handlers.append(new_h) + + converted[log_name].append( + { + 'level': original_log['level'].lower(), + 'handlers': new_handlers + } + ) + + update_logging_config('logging', old_config.logging_config) + update_logging_config('perf_logging', old_config.perf_logging_config) + converted['general'] = [{}] + if hasattr(old_config, 'checks_path'): + converted['general'][0][ + 'check_search_path' + ] = old_config.checks_path + + if hasattr(old_config, 'checks_path_recurse'): + converted['general'][0][ + 'check_search_recursive' + ] = old_config.checks_path_recurse + + if converted['general'] == [{}]: + del converted['general'] + + # Validate the converted file + schema_filename = os.path.join(reframe.INSTALL_PREFIX, + 'schemas', 'config.json') + + # We let the following statements raise, because if they do, that's a BUG + with open(schema_filename) as fp: + schema = json.loads(fp.read()) + + jsonschema.validate(converted, schema) + with tempfile.NamedTemporaryFile(mode='w', delete=False) as fp: + fp.write(f"#\n# This file was automatically generated " + f"by ReFrame based on '{filename}'.\n#\n\n") + fp.write(f'site_configuration = {util.ppretty(converted)}\n') + + return fp.name diff --git a/reframe/utility/__init__.py b/reframe/utility/__init__.py index 070a4eed95..b16a0632ca 100644 --- a/reframe/utility/__init__.py +++ b/reframe/utility/__init__.py @@ -126,6 +126,64 @@ def toalphanum(s): return re.sub(r'\W', '_', s) +def ppretty(value, htchar=' ', lfchar='\n', indent=4, basic_offset=0): + '''Format string of dictionaries, lists and tuples + + :arg value: The value to be formatted. + :arg htchar: Horizontal-tab character. + :arg lfchar: Linefeed character. + :arg indent: Number of htchar characters for every indentation level. + :arg basic_offset: Basic offset for the representation, any additional + indentation space is added to the ``basic_offset``. + :returns: a formatted string of the ``value``. + ''' + + nlch = lfchar + htchar * indent * (basic_offset + 1) + if isinstance(value, tuple): + if value == (): + return '()' + + items = [ + nlch + ppretty(item, htchar, lfchar, indent, basic_offset + 1) + for item in value + ] + return '(%s)' % (','.join(items) + lfchar + + htchar * indent * basic_offset) + elif isinstance(value, list): + if value == []: + return '[]' + + items = [ + nlch + ppretty(item, htchar, lfchar, indent, basic_offset + 1) + for item in value + ] + return '[%s]' % (','.join(items) + lfchar + + htchar * indent * basic_offset) + elif isinstance(value, dict): + if value == {}: + return '{}' + + items = [ + nlch + repr(key) + ': ' + + ppretty(value[key], htchar, lfchar, indent, basic_offset + 1) + for key in value + ] + return '{%s}' % (','.join(items) + lfchar + + htchar * indent * basic_offset) + elif isinstance(value, set): + if value == set(): + return 'set()' + + items = [ + nlch + ppretty(item, htchar, lfchar, indent, basic_offset + 1) + for item in value + ] + return '{%s}' % (','.join(items) + lfchar + + htchar * indent * basic_offset) + else: + return repr(value) + + class ScopedDict(UserDict): '''This is a special dict that imposes scopes on its keys. diff --git a/schemas/config.json b/schemas/config.json index c837c2aea4..6378f2232d 100644 --- a/schemas/config.json +++ b/schemas/config.json @@ -147,7 +147,7 @@ "descr": {"type": "string"}, "scheduler": { "type": "string", - "enum": ["local", "pbs", "slurm", "squeue"] + "enum": ["local", "pbs", "slurm", "squeue", "torque"] }, "launcher": { "type": "string", @@ -310,7 +310,7 @@ "modes": { "type": "array", "items": { - "type": "object", + "type": "object", "properties": { "name": {"type": "string"}, "options": { @@ -350,13 +350,13 @@ "general/check_search_recursive": "true", "general/target_systems": ["*"], "perf_logging/target_systems": ["*"], - "logging/handlers/level": "DEBUG", + "logging/handlers/level": "debug", "logging/handlers/file/append": false, "logging/handlers/file/timestamp": false, "logging/handlers/stream/name": "stdout", "logging/handlers/syslog/socktype": "udp", "logging/handlers/syslog/facility": "user", - "logging/level": "INFO", + "logging/level": "info", "logging/target_systems": ["*"], "modes/target_systems": ["*"], "schedulers/job_submit_timeout": 60, diff --git a/schemas/settings.py b/schemas/settings.py index c95ed050f9..0f2d1b7c0c 100644 --- a/schemas/settings.py +++ b/schemas/settings.py @@ -68,7 +68,7 @@ }, { 'name': 'sys0', - 'descr': 'System for testing check dependencies', + 'descr': 'System for checking test dependencies', 'hostnames': [r'sys\d+'], 'partitions': [ { diff --git a/tools/convert_config.py b/tools/convert_config.py new file mode 100644 index 0000000000..397b3dba6b --- /dev/null +++ b/tools/convert_config.py @@ -0,0 +1,32 @@ +# Copyright 2016-2020 Swiss National Supercomputing Centre (CSCS/ETH Zurich) +# ReFrame Project Developers. See the top-level LICENSE file for details. +# +# SPDX-License-Identifier: BSD-3-Clause + +import os +import sys + +sys.path.append(os.path.join(os.path.dirname(__file__), '..')) + +import reframe.core.config as config # noqa: F401, F403 + + +if __name__ == '__main__': + try: + old_config = sys.argv[1] + except IndexError: + print(f'{sys.argv[0]}: too few arguments', file=sys.stderr) + print(f'Usage: {sys.argv[0]} OLD_CONFIG_FILE', file=sys.stderr) + sys.exit(1) + + try: + new_config = config.convert_old_config(old_config) + except Exception as e: + print(f'{sys.argv[0]}: could not convert file: {e}', + file=sys.stderr) + sys.exit(1) + + print( + f"Conversion successful! " + f"Please find the converted file at '{new_config}'." + ) diff --git a/unittests/resources/settings.py b/unittests/resources/settings.py index a2a65550c2..d3ddc060a7 100644 --- a/unittests/resources/settings.py +++ b/unittests/resources/settings.py @@ -66,7 +66,7 @@ class ReframeSettings: }, 'sys0': { # System used for dependency checking - 'descr': 'System for test dependencies unit tests', + 'descr': 'System for checking test dependencies', 'hostnames': [r'sys\d+'], 'partitions': { 'p0': { diff --git a/unittests/test_utility.py b/unittests/test_utility.py index eeb148dd25..a936468fd8 100644 --- a/unittests/test_utility.py +++ b/unittests/test_utility.py @@ -444,6 +444,94 @@ def __repr__(self): assert 'D(...)' in rep +class TestPpretty: + def test_simple_types(self): + assert util.ppretty(1) == repr(1) + assert util.ppretty(1.2) == repr(1.2) + assert util.ppretty('a string') == repr('a string') + assert util.ppretty([]) == '[]' + assert util.ppretty(()) == '()' + assert util.ppretty(set()) == 'set()' + assert util.ppretty({}) == '{}' + assert util.ppretty([1, 2, 3]) == '[\n 1,\n 2,\n 3\n]' + assert util.ppretty((1, 2, 3)) == '(\n 1,\n 2,\n 3\n)' + assert util.ppretty({1, 2, 3}) == '{\n 1,\n 2,\n 3\n}' + assert util.ppretty({'a': 1, 'b': 2}) == ("{\n" + " 'a': 1,\n" + " 'b': 2\n" + "}") + + def test_mixed_types(self): + assert ( + util.ppretty(['a string', 2, 'another string']) == + "[\n" + " 'a string',\n" + " 2,\n" + " 'another string'\n" + "]" + ) + assert util.ppretty({'a': 1, 'b': (2, 3)}) == ("{\n" + " 'a': 1,\n" + " 'b': (\n" + " 2,\n" + " 3\n" + " )\n" + "}") + assert ( + util.ppretty({'a': 1, 'b': {2: {3: 4, 5: {}}}, 'c': 6}) == + "{\n" + " 'a': 1,\n" + " 'b': {\n" + " 2: {\n" + " 3: 4,\n" + " 5: {}\n" + " }\n" + " },\n" + " 'c': 6\n" + "}") + assert ( + util.ppretty({'a': 2, 34: (2, 3), + 'b': [[], [1.2, 3.4], {1, 2}]}) == + "{\n" + " 'a': 2,\n" + " 34: (\n" + " 2,\n" + " 3\n" + " ),\n" + " 'b': [\n" + " [],\n" + " [\n" + " 1.2,\n" + " 3.4\n" + " ],\n" + " {\n" + " 1,\n" + " 2\n" + " }\n" + " ]\n" + "}" + ) + + def test_obj_print(self): + class C: + def __repr__(self): + return '' + + class D: + def __repr__(self): + return '' + + c = C() + d = D() + assert util.ppretty(c) == '' + assert util.ppretty(['a', 'b', c, d]) == ("[\n" + " 'a',\n" + " 'b',\n" + " ,\n" + " \n" + "]") + + class TestChangeDirCtxManager(unittest.TestCase): def setUp(self): self.temp_dir = tempfile.mkdtemp()