diff --git a/ci-scripts/ci-runner.bash b/ci-scripts/ci-runner.bash index 3e4c9c6f4c..8e8af1d3ec 100644 --- a/ci-scripts/ci-runner.bash +++ b/ci-scripts/ci-runner.bash @@ -33,7 +33,7 @@ EOF checked_exec() { - "$@" + echo "[RUN] $@" && "$@" if [ $? -ne 0 ]; then CI_EXITCODE=1 fi @@ -43,7 +43,7 @@ run_tutorial_checks() { cmd="./bin/reframe -C tutorial/config/settings.py \ --save-log-files -r -t tutorial $@" - echo "Running tutorial checks with \`$cmd'" + echo "[INFO] Running tutorial checks with \`$cmd'" checked_exec $cmd } @@ -51,19 +51,10 @@ run_user_checks() { cmd="./bin/reframe -C config/cscs.py --save-log-files \ -r --flex-alloc-nodes=2 -t production|benchmark $@" - echo "Running user checks with \`$cmd'" + echo "[INFO] Running user checks with \`$cmd'" checked_exec $cmd } -run_serial_user_checks() -{ - cmd="./bin/reframe -C config/cscs.py --exec-policy=serial --save-log-files \ --r -t production-serial $@" - echo "Running user checks with \`$cmd'" - checked_exec $cmd -} - - ### Main script ### shortopts="h,g,t,f:,i:,l:,m:" @@ -105,7 +96,7 @@ while [ $# -ne 0 ]; do --) ;; *) - echo "${scriptname}: Unrecognized argument \`$1'" >&2 + echo "[ERROR] ${scriptname}: Unrecognized argument \`$1'" >&2 usage exit 1 ;; esac @@ -144,27 +135,24 @@ fi # Always install our requirements python3 -m venv venv.unittests source venv.unittests/bin/activate +pip install --upgrade pip pip install -r requirements.txt # FIXME: XALT is causing linking problems (see UES-823) module unload xalt -echo "==============" -echo "Loaded Modules" -echo "==============" +echo "[INFO] Loaded Modules" module list - cd ${CI_FOLDER} -echo "Running regression on $(hostname) in ${CI_FOLDER}" +echo "[INFO] Running unit tests on $(hostname) in ${CI_FOLDER}" if [ $CI_GENERIC -eq 1 ]; then # Run unit tests for the public release - echo "========================================" - echo "Running unit tests with generic settings" - echo "========================================" - checked_exec ./test_reframe.py -W=error::reframe.core.exceptions.ReframeDeprecationWarning - checked_exec ! ./bin/reframe.py -W=error::reframe.core.exceptions.ReframeDeprecationWarning --system=generic -l 2>&1 | \ + echo "[INFO] Running unit tests with generic settings" + checked_exec ./test_reframe.py \ + -W=error::reframe.core.exceptions.ReframeDeprecationWarning -ra + checked_exec ! ./bin/reframe.py --system=generic -l 2>&1 | \ grep -- '--- Logging error ---' elif [ $CI_TUTORIAL -eq 1 ]; then # Run tutorial checks @@ -173,46 +161,51 @@ elif [ $CI_TUTORIAL -eq 1 ]; then grep -e '^tutorial/(?!config/).*\.py') ) if [ ${#tutorialchecks[@]} -ne 0 ]; then - tutorialchecks_path="-c $(IFS=: eval 'echo "${tutorialchecks[*]}"')" + tutorialchecks_path="" + for check in ${tutorialchecks[@]}; do + tutorialchecks_path="${tutorialchecks_path} -c ${check}" + done - echo "========================" - echo "Modified tutorial checks" - echo "========================" + echo "[INFO] Modified tutorial checks" echo ${tutorialchecks_path} - for i in ${!invocations[@]}; do run_tutorial_checks ${tutorialchecks_path} ${invocations[i]} done fi else - # Performing the unittests - echo "==================" - echo "Running unit tests" - echo "==================" - - checked_exec ./test_reframe.py -W=error::reframe.core.exceptions.ReframeDeprecationWarning --rfm-user-config=config/cscs-ci.py - + # Run unit tests with the scheduler backends + tempdir=$(mktemp -d -p $SCRATCH) if [[ $(hostname) =~ dom ]]; then PATH_save=$PATH - for backend in pbs torque; do - echo "==================================" - echo "Running unit tests with ${backend}" - echo "==================================" - export PATH=/apps/dom/UES/karakasv/slurm-wrappers/bin:$PATH - checked_exec ./test_reframe.py -W=error::reframe.core.exceptions.ReframeDeprecationWarning --rfm-user-config=config/cscs-${backend}.py + export PATH=/apps/dom/UES/karakasv/slurm-wrappers/bin:$PATH + for backend in slurm pbs torque; do + echo "[INFO] Running unit tests with ${backend}" + checked_exec ./test_reframe.py --rfm-user-config=config/cscs-ci.py \ + -W=error::reframe.core.exceptions.ReframeDeprecationWarning \ + --rfm-user-system=dom:${backend} --basetemp=$tempdir -ra done export PATH=$PATH_save + else + echo "[INFO] Running unit tests" + checked_exec ./test_reframe.py --rfm-user-config=config/cscs-ci.py \ + -W=error::reframe.core.exceptions.ReframeDeprecationWarning \ + --basetemp=$tempdir -ra + fi + + if [ $CI_EXITCODE -eq 0 ]; then + /bin/rm -rf $tempdir fi # Find modified or added user checks userchecks=( $(git diff origin/master...HEAD --name-only --oneline --no-merges | \ grep -e '^cscs-checks/.*\.py') ) if [ ${#userchecks[@]} -ne 0 ]; then - userchecks_path="-c $(IFS=: eval 'echo "${userchecks[*]}"')" + userchecks_path="" + for check in ${userchecks[@]}; do + userchecks_path="${userchecks_path} -c ${check}" + done - echo "====================" - echo "Modified user checks" - echo "====================" + echo "[INFO] Modified user checks" echo ${userchecks_path} # @@ -220,8 +213,8 @@ else # for i in ${!invocations[@]}; do run_user_checks ${userchecks_path} ${invocations[i]} - run_serial_user_checks ${userchecks_path} ${invocations[i]} done fi fi +deactivate exit $CI_EXITCODE diff --git a/config/cscs-ci.py b/config/cscs-ci.py index 82ce19cad9..8e060eed6e 100644 --- a/config/cscs-ci.py +++ b/config/cscs-ci.py @@ -4,188 +4,247 @@ # SPDX-License-Identifier: BSD-3-Clause # -# CSCS ReFrame CI settings +# CSCS CI settings # - -class ReframeSettings: - job_poll_intervals = [1, 2, 3] - job_submit_timeout = 60 - checks_path = ['checks/'] - checks_path_recurse = True - site_configuration = { - 'systems': { - 'daint': { - 'descr': 'Piz Daint CI nodes', - 'hostnames': ['daint'], - 'modules_system': 'tmod', - 'resourcesdir': '/apps/common/UES/reframe/resources', - 'partitions': { - 'gpu': { - 'scheduler': 'nativeslurm', - 'modules': ['daint-gpu'], - 'access': ['--constraint=gpu', '--partition=cscsci'], - 'environs': ['PrgEnv-cray'], - 'descr': 'Hybrid nodes (Haswell/P100)', - 'max_jobs': 100, - 'resources': { - 'switches': ['--switches={num_switches}'] +site_configuration = { + 'systems': [ + { + 'name': 'daint', + 'descr': 'Piz Daint CI nodes', + 'hostnames': [ + 'daint' + ], + 'modules_system': 'tmod', + 'resourcesdir': '/apps/common/UES/reframe/resources', + 'partitions': [ + { + 'name': 'gpu', + 'scheduler': 'slurm', + 'modules': [ + 'daint-gpu' + ], + 'access': [ + '--constraint=gpu', + '--partition=cscsci' + ], + 'environs': [ + 'builtin' + ], + 'descr': 'Hybrid nodes (Haswell/P100)', + 'max_jobs': 100, + 'resources': [ + { + 'name': 'switches', + 'options': [ + '--switches={num_switches}' + ] } - }, + ], + 'launcher': 'srun' } - }, - 'dom': { - 'descr': 'Dom TDS', - 'hostnames': ['dom'], - 'modules_system': 'tmod', - 'resourcesdir': '/apps/common/UES/reframe/resources', - 'partitions': { - 'gpu': { - 'scheduler': 'nativeslurm', - 'modules': ['daint-gpu'], - 'access': ['--constraint=gpu'], - 'environs': ['PrgEnv-cray'], - 'descr': 'Hybrid nodes (Haswell/P100)', - 'max_jobs': 100, - 'resources': { - 'switches': ['--switches={num_switches}'] + ] + }, + { + 'name': 'dom', + 'descr': 'Dom TDS', + 'hostnames': [ + 'dom' + ], + 'modules_system': 'tmod', + 'resourcesdir': '/apps/common/UES/reframe/resources', + 'partitions': [ + { + 'name': 'slurm', + 'scheduler': 'slurm', + 'modules': [ + 'daint-gpu' + ], + 'access': [ + '--constraint=gpu' + ], + 'environs': [ + 'builtin' + ], + 'descr': 'Hybrid nodes (Haswell/P100)', + 'max_jobs': 100, + 'resources': [ + { + 'name': 'switches', + 'options': [ + '--switches={num_switches}' + ] } - }, + ], + 'launcher': 'srun' + }, + { + 'name': 'pbs', + 'scheduler': 'pbs', + 'modules': [ + 'daint-gpu' + ], + 'access': [ + 'proc=gpu' + ], + 'environs': [ + 'builtin' + ], + 'descr': 'Hybrid nodes (Haswell/P100)', + 'max_jobs': 100, + 'launcher': 'mpiexec' + }, + { + 'name': 'torque', + 'scheduler': 'torque', + 'modules': [ + 'daint-gpu' + ], + 'access': [ + '-l proc=gpu' + ], + 'environs': [ + 'builtin' + ], + 'descr': 'Hybrid nodes (Haswell/P100)', + 'max_jobs': 100, + 'launcher': 'mpiexec' } - }, - 'kesch': { - 'descr': 'Kesch MCH', - 'hostnames': [r'keschln-\d+'], - 'modules_system': 'tmod', - 'resourcesdir': '/apps/common/UES/reframe/resources', - 'partitions': { - 'cn': { - 'scheduler': 'nativeslurm', - 'access': ['--partition=cn-regression'], - 'environs': ['PrgEnv-cray'], - 'descr': 'Kesch compute nodes', - 'resources': { - '_rfm_gpu': ['--gres=gpu:{num_gpus_per_node}'], + ] + }, + { + 'name': 'kesch', + 'descr': 'Kesch MCH', + 'hostnames': [ + r'keschln-\d+' + ], + 'modules_system': 'tmod', + 'resourcesdir': '/apps/common/UES/reframe/resources', + 'partitions': [ + { + 'name': 'cn', + 'scheduler': 'slurm', + 'access': [ + '--partition=cn-regression' + ], + 'environs': [ + 'builtin' + ], + 'descr': 'Kesch compute nodes', + 'resources': [ + { + 'name': '_rfm_gpu', + 'options': [ + '--gres=gpu:{num_gpus_per_node}' + ] } - } + ], + 'launcher': 'srun' } - }, - 'tsa': { - 'descr': 'Tsa MCH', - 'hostnames': [r'tsa-\w+\d+'], - 'modules_system': 'tmod', - 'resourcesdir': '/apps/common/UES/reframe/resources', - 'partitions': { - 'cn': { - 'scheduler': 'nativeslurm', - 'access': ['--partition=cn-regression'], - 'environs': ['builtin-gcc'], - 'descr': 'Tsa compute nodes', - 'max_jobs': 10, - 'resources': { - '_rfm_gpu': ['--gres=gpu:{num_gpus_per_node}'], + ] + }, + { + 'name': 'tsa', + 'descr': 'Tsa MCH', + 'hostnames': [ + r'tsa-\w+\d+' + ], + 'modules_system': 'tmod', + 'resourcesdir': '/apps/common/UES/reframe/resources', + 'partitions': [ + { + 'name': 'cn', + 'scheduler': 'slurm', + 'access': [ + '--partition=cn-regression' + ], + 'environs': [ + 'builtin' + ], + 'descr': 'Tsa compute nodes', + 'max_jobs': 10, + 'resources': [ + { + 'name': '_rfm_gpu', + 'options': [ + '--gres=gpu:{num_gpus_per_node}' + ] } - } + ], + 'launcher': 'srun' } - }, - 'generic': { - 'descr': 'Generic example system', - 'partitions': { - 'login': { - 'scheduler': 'local', - 'modules': [], - 'access': [], - 'environs': ['builtin-gcc'], - 'descr': 'Login nodes' - } + ] + }, + { + 'name': 'generic', + 'descr': 'Generic example system', + 'partitions': [ + { + 'name': 'default', + 'scheduler': 'local', + 'modules': [], + 'access': [], + 'environs': [ + 'builtin' + ], + 'descr': 'Login nodes', + 'launcher': 'local' } - } + ], + 'hostnames': ['.*'] + } + ], + 'environments': [ + { + 'name': 'builtin', + 'cc': 'cc', + 'cxx': '', + 'ftn': '' }, - - 'environments': { - '*': { - 'PrgEnv-cray': { - 'modules': ['PrgEnv-cray'], - }, - - 'PrgEnv-gnu': { - 'modules': ['PrgEnv-gnu'], - }, - - 'PrgEnv-intel': { - 'modules': ['PrgEnv-intel'], - }, - - 'PrgEnv-pgi': { - 'modules': ['PrgEnv-pgi'], + ], + 'logging': [ + { + 'level': 'debug', + 'handlers': [ + { + 'type': 'file', + 'name': 'reframe.log', + 'level': 'debug', + 'format': '[%(asctime)s] %(levelname)s: %(check_info)s: %(message)s', # noqa: E501 + 'append': False }, - - 'builtin': { - 'cc': 'cc', - 'cxx': '', - 'ftn': '', + { + 'type': 'stream', + 'name': 'stdout', + 'level': 'info', + 'format': '%(message)s' }, - - 'builtin-gcc': { - 'cc': 'gcc', - 'cxx': 'g++', - 'ftn': 'gfortran', + { + 'type': 'file', + 'name': 'reframe.out', + 'level': 'info', + 'format': '%(message)s', + 'append': False + } + ], + 'handlers_perflog': [ + { + 'type': 'filelog', + 'prefix': '%(check_system)s/%(check_partition)s', + 'level': 'info', + 'format': '%(check_job_completion_time)s|reframe %(version)s|%(check_info)s|jobid=%(check_jobid)s|num_tasks=%(check_num_tasks)s|%(check_perf_var)s=%(check_perf_value)s|ref=%(check_perf_ref)s (l=%(check_perf_lower_thres)s, u=%(check_perf_upper_thres)s)|%(check_perf_unit)s', # noqa: E501 + 'datefmt': '%FT%T%:z', + 'append': True } - } + ] } - } - - logging_config = { - 'level': 'DEBUG', - 'handlers': [ - { - 'type': 'file', - 'name': 'reframe.log', - 'level': 'DEBUG', - 'format': '[%(asctime)s] %(levelname)s: ' - '%(check_info)s: %(message)s', - 'append': False, - }, - - # Output handling - { - 'type': 'stream', - 'name': 'stdout', - 'level': 'INFO', - 'format': '%(message)s' - }, - { - 'type': 'file', - 'name': 'reframe.out', - 'level': 'INFO', - 'format': '%(message)s', - 'append': False, - } - ] - } - - perf_logging_config = { - 'level': 'DEBUG', - 'handlers': [ - { - 'type': 'filelog', - 'prefix': '%(check_system)s/%(check_partition)s', - 'level': 'INFO', - 'format': ( - '%(check_job_completion_time)s|reframe %(version)s|' - '%(check_info)s|jobid=%(check_jobid)s|' - 'num_tasks=%(check_num_tasks)s|' - '%(check_perf_var)s=%(check_perf_value)s|' - 'ref=%(check_perf_ref)s ' - '(l=%(check_perf_lower_thres)s, ' - 'u=%(check_perf_upper_thres)s)|' - '%(check_perf_unit)s' - ), - 'datefmt': '%FT%T%:z', - 'append': True - } - ] - } - - -settings = ReframeSettings() + ], + 'general': [ + { + 'check_search_path': [ + 'checks/' + ], + 'check_search_recursive': True + } + ] +} diff --git a/config/cscs-pbs.py b/config/cscs-pbs.py deleted file mode 100644 index 9854dc1b75..0000000000 --- a/config/cscs-pbs.py +++ /dev/null @@ -1,155 +0,0 @@ -# Copyright 2016-2020 Swiss National Supercomputing Centre (CSCS/ETH Zurich) -# ReFrame Project Developers. See the top-level LICENSE file for details. -# -# SPDX-License-Identifier: BSD-3-Clause - -# -# Minimal CSCS configuration for testing the PBS backend -# - - -class ReframeSettings: - job_poll_intervals = [1, 2, 3] - job_submit_timeout = 60 - checks_path = ['checks/'] - checks_path_recurse = True - site_configuration = { - 'systems': { - 'dom': { - 'descr': 'Dom TDS', - 'hostnames': ['dom'], - 'modules_system': 'tmod', - 'resourcesdir': '/apps/common/UES/reframe/resources', - 'partitions': { - 'login': { - 'scheduler': 'local', - 'modules': [], - 'access': [], - 'environs': ['PrgEnv-cray', 'PrgEnv-gnu', - 'PrgEnv-intel', 'PrgEnv-pgi'], - 'descr': 'Login nodes', - 'max_jobs': 4 - }, - - 'gpu': { - 'scheduler': 'pbs+mpiexec', - 'modules': ['daint-gpu'], - 'access': ['proc=gpu'], - 'environs': ['PrgEnv-cray', 'PrgEnv-gnu', - 'PrgEnv-intel', 'PrgEnv-pgi'], - 'descr': 'Hybrid nodes (Haswell/P100)', - 'max_jobs': 100, - }, - - 'mc': { - 'scheduler': 'pbs+mpiexec', - 'modules': ['daint-mc'], - 'access': ['proc=mc'], - 'environs': ['PrgEnv-cray', 'PrgEnv-gnu', - 'PrgEnv-intel', 'PrgEnv-pgi'], - 'descr': 'Multicore nodes (Broadwell)', - 'max_jobs': 100, - }, - } - }, - - 'generic': { - 'descr': 'Generic example system', - 'partitions': { - 'login': { - 'scheduler': 'local', - 'modules': [], - 'access': [], - 'environs': ['builtin-gcc'], - 'descr': 'Login nodes' - } - } - } - }, - - 'environments': { - '*': { - 'PrgEnv-cray': { - 'modules': ['PrgEnv-cray'], - }, - - 'PrgEnv-gnu': { - 'modules': ['PrgEnv-gnu'], - }, - - 'PrgEnv-intel': { - 'modules': ['PrgEnv-intel'], - }, - - 'PrgEnv-pgi': { - 'modules': ['PrgEnv-pgi'], - }, - - 'builtin': { - 'cc': 'cc', - 'cxx': '', - 'ftn': '', - }, - - 'builtin-gcc': { - 'cc': 'gcc', - 'cxx': 'g++', - 'ftn': 'gfortran', - } - } - }, - } - - logging_config = { - 'level': 'DEBUG', - 'handlers': [ - { - 'type': 'file', - 'name': 'reframe.log', - 'level': 'DEBUG', - 'format': '[%(asctime)s] %(levelname)s: ' - '%(check_info)s: %(message)s', - 'append': False, - }, - - # Output handling - { - 'type': 'stream', - 'name': 'stdout', - 'level': 'INFO', - 'format': '%(message)s' - }, - { - 'type': 'file', - 'name': 'reframe.out', - 'level': 'INFO', - 'format': '%(message)s', - 'append': False, - } - ] - } - - perf_logging_config = { - 'level': 'DEBUG', - 'handlers': [ - { - 'type': 'filelog', - 'prefix': '%(check_system)s/%(check_partition)s', - 'level': 'INFO', - 'format': ( - '%(check_job_completion_time)s|reframe %(version)s|' - '%(check_info)s|jobid=%(check_jobid)s|' - 'num_tasks=%(check_num_tasks)s|' - '%(check_perf_var)s=%(check_perf_value)s|' - 'ref=%(check_perf_ref)s ' - '(l=%(check_perf_lower_thres)s, ' - 'u=%(check_perf_upper_thres)s)' - ), - 'datefmt': '%FT%T%:z', - 'append': True - } - ] - } - - -settings = ReframeSettings() diff --git a/config/cscs-torque.py b/config/cscs-torque.py deleted file mode 100644 index 563b12970f..0000000000 --- a/config/cscs-torque.py +++ /dev/null @@ -1,155 +0,0 @@ -# Copyright 2016-2020 Swiss National Supercomputing Centre (CSCS/ETH Zurich) -# ReFrame Project Developers. See the top-level LICENSE file for details. -# -# SPDX-License-Identifier: BSD-3-Clause - -# -# Minimal CSCS configuration for testing the Torque backend -# - - -class ReframeSettings: - job_poll_intervals = [1, 2, 3] - job_submit_timeout = 60 - checks_path = ['checks/'] - checks_path_recurse = True - site_configuration = { - 'systems': { - 'dom': { - 'descr': 'Dom TDS', - 'hostnames': ['dom'], - 'modules_system': 'tmod', - 'resourcesdir': '/apps/common/UES/reframe/resources', - 'partitions': { - 'login': { - 'scheduler': 'local', - 'modules': [], - 'access': [], - 'environs': ['PrgEnv-cray', 'PrgEnv-gnu', - 'PrgEnv-intel', 'PrgEnv-pgi'], - 'descr': 'Login nodes', - 'max_jobs': 4 - }, - - 'gpu': { - 'scheduler': 'torque+mpiexec', - 'modules': ['daint-gpu'], - 'access': ['-l proc=gpu'], - 'environs': ['PrgEnv-cray', 'PrgEnv-gnu', - 'PrgEnv-intel', 'PrgEnv-pgi'], - 'descr': 'Hybrid nodes (Haswell/P100)', - 'max_jobs': 100, - }, - - 'mc': { - 'scheduler': 'torque+mpiexec', - 'modules': ['daint-mc'], - 'access': ['-l proc=mc'], - 'environs': ['PrgEnv-cray', 'PrgEnv-gnu', - 'PrgEnv-intel', 'PrgEnv-pgi'], - 'descr': 'Multicore nodes (Broadwell)', - 'max_jobs': 100, - }, - } - }, - - 'generic': { - 'descr': 'Generic example system', - 'partitions': { - 'login': { - 'scheduler': 'local', - 'modules': [], - 'access': [], - 'environs': ['builtin-gcc'], - 'descr': 'Login nodes' - } - } - } - }, - - 'environments': { - '*': { - 'PrgEnv-cray': { - 'modules': ['PrgEnv-cray'], - }, - - 'PrgEnv-gnu': { - 'modules': ['PrgEnv-gnu'], - }, - - 'PrgEnv-intel': { - 'modules': ['PrgEnv-intel'], - }, - - 'PrgEnv-pgi': { - 'modules': ['PrgEnv-pgi'], - }, - - 'builtin': { - 'cc': 'cc', - 'cxx': '', - 'ftn': '', - }, - - 'builtin-gcc': { - 'cc': 'gcc', - 'cxx': 'g++', - 'ftn': 'gfortran', - } - } - }, - } - - logging_config = { - 'level': 'DEBUG', - 'handlers': [ - { - 'type': 'file', - 'name': 'reframe.log', - 'level': 'DEBUG', - 'format': '[%(asctime)s] %(levelname)s: ' - '%(check_info)s: %(message)s', - 'append': False, - }, - - # Output handling - { - 'type': 'stream', - 'name': 'stdout', - 'level': 'INFO', - 'format': '%(message)s' - }, - { - 'type': 'file', - 'name': 'reframe.out', - 'level': 'INFO', - 'format': '%(message)s', - 'append': False, - } - ] - } - - perf_logging_config = { - 'level': 'DEBUG', - 'handlers': [ - { - 'type': 'filelog', - 'prefix': '%(check_system)s/%(check_partition)s', - 'level': 'INFO', - 'format': ( - '%(check_job_completion_time)s|reframe %(version)s|' - '%(check_info)s|jobid=%(check_jobid)s|' - 'num_tasks=%(check_num_tasks)s|' - '%(check_perf_var)s=%(check_perf_value)s|' - 'ref=%(check_perf_ref)s ' - '(l=%(check_perf_lower_thres)s, ' - 'u=%(check_perf_upper_thres)s)' - ), - 'datefmt': '%FT%T%:z', - 'append': True - } - ] - } - - -settings = ReframeSettings() diff --git a/config/cscs.py b/config/cscs.py index 28c5ae5739..2730934dfb 100644 --- a/config/cscs.py +++ b/config/cscs.py @@ -2,647 +2,1009 @@ # ReFrame Project Developers. See the top-level LICENSE file for details. # # SPDX-License-Identifier: BSD-3-Clause - # -# CSCS ReFrame settings +# ReFrame CSCS settings # - -class ReframeSettings: - job_poll_intervals = [1, 2, 3] - job_submit_timeout = 60 - checks_path = ['checks/'] - checks_path_recurse = True - site_configuration = { - 'systems': { - 'ault': { - 'descr': 'Ault TDS', - 'hostnames': ['ault'], - 'modules_system': 'lmod', - 'resourcesdir': '/apps/common/UES/reframe/resources', - 'partitions': { - 'login': { - 'scheduler': 'local', - 'environs': ['builtin', 'PrgEnv-gnu'], - 'descr': 'Login nodes', - 'max_jobs': 4 - }, - 'amdv100': { - 'scheduler': 'nativeslurm', - 'access': ['-pamdv100'], - 'environs': ['builtin', 'PrgEnv-gnu'], - 'descr': 'AMD Naples 32c + 2x NVIDIA V100', - 'max_jobs': 100, - }, - 'amdvega': { - 'scheduler': 'nativeslurm', - 'access': ['-pamdvega'], - 'environs': ['builtin', 'PrgEnv-gnu'], - 'descr': 'AMD Naples 32c + 3x AMD GFX900', - 'max_jobs': 100, - }, - 'intelv100': { - 'scheduler': 'nativeslurm', - 'access': ['-pintelv100'], - 'environs': ['builtin', 'PrgEnv-gnu'], - 'descr': 'Intel Skylake 36c + 4x NVIDIA V100', - 'max_jobs': 100, - }, - 'intel': { - 'scheduler': 'nativeslurm', - 'access': ['-pintel'], - 'environs': ['builtin', 'PrgEnv-gnu'], - 'descr': 'Intel Skylake 36c', - 'max_jobs': 100, - } +site_configuration = { + 'systems': [ + { + 'name': 'ault', + 'descr': 'Ault TDS', + 'hostnames': [ + 'ault' + ], + 'modules_system': 'lmod', + 'resourcesdir': '/apps/common/UES/reframe/resources', + 'partitions': [ + { + 'name': 'login', + 'scheduler': 'local', + 'environs': [ + 'builtin', + 'PrgEnv-gnu' + ], + 'descr': 'Login nodes', + 'max_jobs': 4, + 'launcher': 'local' + }, + { + 'name': 'amdv100', + 'scheduler': 'slurm', + 'access': [ + '-pamdv100' + ], + 'environs': [ + 'builtin', + 'PrgEnv-gnu' + ], + 'descr': 'AMD Naples 32c + 2x NVIDIA V100', + 'max_jobs': 100, + 'launcher': 'srun' + }, + { + 'name': 'amdvega', + 'scheduler': 'slurm', + 'access': [ + '-pamdvega' + ], + 'environs': [ + 'builtin', + 'PrgEnv-gnu' + ], + 'descr': 'AMD Naples 32c + 3x AMD GFX900', + 'max_jobs': 100, + 'launcher': 'srun' + }, + { + 'name': 'intelv100', + 'scheduler': 'slurm', + 'access': [ + '-pintelv100' + ], + 'environs': [ + 'builtin', + 'PrgEnv-gnu' + ], + 'descr': 'Intel Skylake 36c + 4x NVIDIA V100', + 'max_jobs': 100, + 'launcher': 'srun' + }, + { + 'name': 'intel', + 'scheduler': 'slurm', + 'access': [ + '-pintel' + ], + 'environs': [ + 'builtin', + 'PrgEnv-gnu' + ], + 'descr': 'Intel Skylake 36c', + 'max_jobs': 100, + 'launcher': 'srun' } - }, - - 'tave': { - 'descr': 'Grand Tave', - 'hostnames': ['tave'], - 'modules_system': 'tmod', - 'resourcesdir': '/apps/common/UES/reframe/resources', - 'partitions': { - 'login': { - 'scheduler': 'local', - 'environs': ['builtin', 'PrgEnv-cray', 'PrgEnv-gnu', - 'PrgEnv-intel', 'PrgEnv-pgi'], - 'descr': 'Login nodes', - 'max_jobs': 4 - }, - 'compute': { - 'scheduler': 'nativeslurm', - 'environs': ['PrgEnv-cray', 'PrgEnv-gnu', - 'PrgEnv-intel', 'PrgEnv-pgi'], - 'descr': 'Intel Xeon Phi', - 'max_jobs': 100, - } + ] + }, + { + 'name': 'tave', + 'descr': 'Grand Tave', + 'hostnames': [ + 'tave' + ], + 'modules_system': 'tmod', + 'resourcesdir': '/apps/common/UES/reframe/resources', + 'partitions': [ + { + 'name': 'login', + 'scheduler': 'local', + 'environs': [ + 'builtin', + 'PrgEnv-cray', + 'PrgEnv-gnu', + 'PrgEnv-intel', + 'PrgEnv-pgi' + ], + 'descr': 'Login nodes', + 'max_jobs': 4, + 'launcher': 'local' + }, + { + 'name': 'compute', + 'scheduler': 'slurm', + 'environs': [ + 'PrgEnv-cray', + 'PrgEnv-gnu', + 'PrgEnv-intel', + 'PrgEnv-pgi' + ], + 'descr': 'Intel Xeon Phi', + 'max_jobs': 100, + 'launcher': 'srun' } - }, - - 'daint': { - 'descr': 'Piz Daint', - 'hostnames': ['daint'], - 'modules_system': 'tmod', - 'resourcesdir': '/apps/common/UES/reframe/resources', - 'partitions': { - 'login': { - 'scheduler': 'local', - 'modules': [], - 'access': [], - 'environs': ['builtin', 'PrgEnv-cray', 'PrgEnv-gnu', - 'PrgEnv-intel', 'PrgEnv-pgi'], - 'descr': 'Login nodes', - 'max_jobs': 4 - }, - - 'gpu': { - 'scheduler': 'nativeslurm', - 'container_platforms': { - 'Sarus': { - 'modules': ['sarus'] - }, - 'Singularity': { - 'modules': ['singularity'] - } + ] + }, + { + 'name': 'daint', + 'descr': 'Piz Daint', + 'hostnames': [ + 'daint' + ], + 'modules_system': 'tmod', + 'resourcesdir': '/apps/common/UES/reframe/resources', + 'partitions': [ + { + 'name': 'login', + 'scheduler': 'local', + 'modules': [], + 'access': [], + 'environs': [ + 'builtin', + 'PrgEnv-cray', + 'PrgEnv-gnu', + 'PrgEnv-intel', + 'PrgEnv-pgi' + ], + 'descr': 'Login nodes', + 'max_jobs': 4, + 'launcher': 'local' + }, + { + 'name': 'gpu', + 'scheduler': 'slurm', + 'container_platforms': [ + { + 'type': 'Sarus', + 'modules': [ + 'sarus' + ] }, - 'modules': ['daint-gpu'], - 'access': ['--constraint=gpu'], - 'environs': ['builtin', 'PrgEnv-cray', 'PrgEnv-gnu', - 'PrgEnv-intel', 'PrgEnv-pgi'], - 'descr': 'Hybrid nodes (Haswell/P100)', - 'max_jobs': 100, - 'resources': { - 'switches': ['--switches={num_switches}'], - 'gres': ['--gres={gres}'] + { + 'type': 'Singularity', + 'modules': [ + 'singularity' + ] } - }, - - 'mc': { - 'scheduler': 'nativeslurm', - 'container_platforms': { - 'Sarus': { - 'modules': ['sarus'] - }, - 'Singularity': { - 'modules': ['singularity'] - } + ], + 'modules': [ + 'daint-gpu' + ], + 'access': [ + '--constraint=gpu' + ], + 'environs': [ + 'builtin', + 'PrgEnv-cray', + 'PrgEnv-gnu', + 'PrgEnv-intel', + 'PrgEnv-pgi' + ], + 'descr': 'Hybrid nodes (Haswell/P100)', + 'max_jobs': 100, + 'resources': [ + { + 'name': 'switches', + 'options': [ + '--switches={num_switches}' + ] }, - 'modules': ['daint-mc'], - 'access': ['--constraint=mc'], - 'environs': ['builtin', 'PrgEnv-cray', 'PrgEnv-gnu', - 'PrgEnv-intel', 'PrgEnv-pgi'], - 'descr': 'Multicore nodes (Broadwell)', - 'max_jobs': 100, - 'resources': { - 'switches': ['--switches={num_switches}'], - 'gres': ['--gres={gres}'] + { + 'name': 'gres', + 'options': ['--gres={gres}'] } - }, - - 'jupyter_gpu': { - 'scheduler': 'nativeslurm', - 'environs': ['builtin'], - 'access': ['-Cgpu', '--reservation=jupyter_gpu'], - 'descr': 'JupyterHub GPU nodes', - 'max_jobs': 10, - }, - - 'jupyter_mc': { - 'scheduler': 'nativeslurm', - 'environs': ['builtin'], - 'access': ['-Cmc', '--reservation=jupyter_mc'], - 'descr': 'JupyterHub multicore nodes', - 'max_jobs': 10, - } - } - }, - - 'dom': { - 'descr': 'Dom TDS', - 'hostnames': ['dom'], - 'modules_system': 'tmod', - 'resourcesdir': '/apps/common/UES/reframe/resources', - 'partitions': { - # FIXME: temporarily disable PrgEnv-pgi on all partitions - 'login': { - 'scheduler': 'local', - 'modules': [], - 'access': [], - 'environs': [ - 'builtin', 'PrgEnv-cray', 'PrgEnv-cray_classic', - 'PrgEnv-gnu', 'PrgEnv-intel', 'PrgEnv-pgi' - ], - 'descr': 'Login nodes', - 'max_jobs': 4 - }, - - 'gpu': { - 'scheduler': 'nativeslurm', - 'container_platforms': { - 'Sarus': { - 'modules': ['sarus'] - }, - 'Singularity': { - 'modules': ['singularity'] - }, + ], + 'launcher': 'srun' + }, + { + 'name': 'mc', + 'scheduler': 'slurm', + 'container_platforms': [ + { + 'type': 'Sarus', + 'modules': [ + 'sarus' + ] }, - 'modules': ['daint-gpu'], - 'access': ['--constraint=gpu'], - 'environs': ['builtin', 'PrgEnv-cray', - 'PrgEnv-cray_classic', 'PrgEnv-gnu', - 'PrgEnv-intel', 'PrgEnv-pgi'], - 'descr': 'Hybrid nodes (Haswell/P100)', - 'max_jobs': 100, - 'resources': { - 'gres': ['--gres={gres}'] + { + 'type': 'Singularity', + 'modules': [ + 'singularity' + ] } - }, - - 'mc': { - 'scheduler': 'nativeslurm', - 'container_platforms': { - 'Sarus': { - 'modules': ['sarus'] - }, - 'Singularity': { - 'modules': ['singularity'] - }, + ], + 'modules': [ + 'daint-mc' + ], + 'access': [ + '--constraint=mc' + ], + 'environs': [ + 'builtin', + 'PrgEnv-cray', + 'PrgEnv-gnu', + 'PrgEnv-intel', + 'PrgEnv-pgi' + ], + 'descr': 'Multicore nodes (Broadwell)', + 'max_jobs': 100, + 'resources': [ + { + 'name': 'switches', + 'options': [ + '--switches={num_switches}' + ] }, - 'modules': ['daint-mc'], - 'access': ['--constraint=mc'], - 'environs': ['builtin', 'PrgEnv-cray', - 'PrgEnv-cray_classic', 'PrgEnv-gnu', - 'PrgEnv-intel', 'PrgEnv-pgi'], - 'descr': 'Multicore nodes (Broadwell)', - 'max_jobs': 100, - 'resources': { - 'gres': ['--gres={gres}'] - } - }, - - 'jupyter_gpu': { - 'scheduler': 'nativeslurm', - 'environs': ['builtin'], - 'access': ['-Cgpu', '--reservation=jupyter_gpu'], - 'descr': 'JupyterHub GPU nodes', - 'max_jobs': 10, - }, - - 'jupyter_mc': { - 'scheduler': 'nativeslurm', - 'environs': ['builtin'], - 'access': ['-Cmc', '--reservation=jupyter_mc'], - 'descr': 'JupyterHub multicore nodes', - 'max_jobs': 10, - } - } - }, - - 'fulen': { - 'descr': 'Fulen', - 'hostnames': [r'fulen-ln\d+'], - 'modules_system': 'tmod', - 'resourcesdir': '/apps/common/UES/reframe/resources', - 'partitions': { - 'login': { - 'scheduler': 'local', - 'environs': ['PrgEnv-gnu'], - 'descr': 'Login nodes', - 'max_jobs': 1 - }, - - 'normal': { - 'scheduler': 'nativeslurm', - 'environs': ['PrgEnv-gnu'], - 'descr': 'Compute nodes - default partition', - }, - - 'fat': { - 'scheduler': 'nativeslurm', - 'environs': ['PrgEnv-gnu'], - 'access': ['--partition fat'], - 'descr': 'High-memory compute nodes', - }, - - 'gpu': { - 'scheduler': 'nativeslurm', - 'environs': ['PrgEnv-gnu'], - 'access': ['--partition gpu'], - 'descr': 'Hybrid compute nodes', - }, - } - }, - - 'kesch': { - 'descr': 'Kesch MCH', - 'hostnames': [r'keschln-\d+'], - 'modules_system': 'tmod', - 'resourcesdir': '/apps/common/UES/reframe/resources', - 'partitions': { - 'login': { - 'scheduler': 'local', - 'environs': ['PrgEnv-cray', 'PrgEnv-cray-nompi', - 'PrgEnv-pgi', 'PrgEnv-pgi-nompi', - 'PrgEnv-gnu', 'PrgEnv-gnu-nompi'], - 'descr': 'Kesch login nodes', - }, - 'pn': { - 'scheduler': 'nativeslurm', - 'access': ['--partition=pn-regression'], - 'environs': ['PrgEnv-cray', 'PrgEnv-cray-nompi', - 'PrgEnv-pgi', 'PrgEnv-pgi-nompi', - 'PrgEnv-gnu', 'PrgEnv-gnu-nompi'], - 'descr': 'Kesch post-processing nodes' - }, - - 'cn': { - 'scheduler': 'nativeslurm', - 'access': ['--partition=cn-regression'], - 'environs': ['PrgEnv-cray', 'PrgEnv-cray-nompi', - 'PrgEnv-pgi', 'PrgEnv-pgi-nompi', - 'PrgEnv-gnu', 'PrgEnv-gnu-nompi'], - 'descr': 'Kesch compute nodes', - 'resources': { - '_rfm_gpu': ['--gres=gpu:{num_gpus_per_node}'], - } - } - } - }, - - 'arolla': { - 'descr': 'Arolla MCH', - 'hostnames': [r'arolla-\w+\d+'], - 'modules_system': 'tmod', - 'resourcesdir': '/apps/common/UES/reframe/resources', - 'partitions': { - 'login': { - 'scheduler': 'local', - 'environs': ['PrgEnv-pgi', 'PrgEnv-pgi-nompi', - 'PrgEnv-gnu', 'PrgEnv-gnu-nompi'], - 'descr': 'Arolla login nodes', - }, - 'pn': { - 'scheduler': 'nativeslurm', - 'access': ['--partition=pn-regression'], - 'environs': ['PrgEnv-pgi', 'PrgEnv-pgi-nompi', - 'PrgEnv-gnu', 'PrgEnv-gnu-nompi'], - 'descr': 'Arolla post-processing nodes', - }, - 'cn': { - 'scheduler': 'nativeslurm', - 'access': ['--partition=cn-regression'], - 'environs': ['PrgEnv-gnu', 'PrgEnv-gnu-nompi', - 'PrgEnv-pgi', 'PrgEnv-pgi-nompi'], - 'descr': 'Arolla compute nodes', - 'resources': { - '_rfm_gpu': ['--gres=gpu:{num_gpus_per_node}'], - } - } - } - }, - - 'tsa': { - 'descr': 'Tsa MCH', - 'hostnames': [r'tsa-\w+\d+'], - 'modules_system': 'tmod', - 'resourcesdir': '/apps/common/UES/reframe/resources', - 'partitions': { - 'login': { - 'scheduler': 'local', - 'environs': ['PrgEnv-pgi', 'PrgEnv-pgi-nompi', - 'PrgEnv-gnu', 'PrgEnv-gnu-nompi'], - 'descr': 'Tsa login nodes', - }, - 'pn': { - 'scheduler': 'nativeslurm', - 'access': ['--partition=pn-regression'], - 'environs': ['PrgEnv-pgi', 'PrgEnv-pgi-nompi', - 'PrgEnv-gnu', 'PrgEnv-gnu-nompi'], - 'descr': 'Tsa post-processing nodes', - }, - 'cn': { - 'scheduler': 'nativeslurm', - 'access': ['--partition=cn-regression'], - 'environs': ['PrgEnv-gnu', 'PrgEnv-gnu-nompi', - 'PrgEnv-pgi', 'PrgEnv-pgi-nompi'], - 'descr': 'Tsa compute nodes', - 'resources': { - '_rfm_gpu': ['--gres=gpu:{num_gpus_per_node}'], + { + 'name': 'gres', + 'options': ['--gres={gres}'] } - } - } - }, - - 'generic': { - 'descr': 'Generic example system', - 'partitions': { - 'login': { - 'scheduler': 'local', - 'modules': [], - 'access': [], - 'environs': ['builtin-gcc'], - 'descr': 'Login nodes', - } - } - } - }, - - 'environments': { - - 'ault': { - 'PrgEnv-gnu': { - # defaults were gcc/8.3.0, cuda/10.1, openmpi/4.0.0 - 'modules': ['gcc', 'cuda/10.1', 'openmpi'], - 'cc': 'mpicc', - 'cxx': 'mpicxx', - 'ftn': 'mpif90', + ], + 'launcher': 'srun' }, - 'builtin': { - 'cc': 'cc', - 'cxx': '', - 'ftn': '', + { + 'name': 'jupyter_gpu', + 'scheduler': 'slurm', + 'environs': [ + 'builtin' + ], + 'access': [ + '-Cgpu', + '--reservation=jupyter_gpu' + ], + 'descr': 'JupyterHub GPU nodes', + 'max_jobs': 10, + 'launcher': 'srun' }, - 'builtin-gcc': { - 'cc': 'gcc', - 'cxx': 'g++', - 'ftn': 'gfortran', + { + 'name': 'jupyter_mc', + 'scheduler': 'slurm', + 'environs': [ + 'builtin' + ], + 'access': [ + '-Cmc', + '--reservation=jupyter_mc' + ], + 'descr': 'JupyterHub multicore nodes', + 'max_jobs': 10, + 'launcher': 'srun' } - }, - - 'kesch': { - 'PrgEnv-pgi-nompi': { - 'modules': ['PE/17.06', - 'PrgEnv-pgi/18.5'], - 'cc': 'pgcc', - 'cxx': 'pgc++', - 'ftn': 'pgf90', + ] + }, + { + 'name': 'dom', + 'descr': 'Dom TDS', + 'hostnames': [ + 'dom' + ], + 'modules_system': 'tmod', + 'resourcesdir': '/apps/common/UES/reframe/resources', + 'partitions': [ + { + 'name': 'login', + 'scheduler': 'local', + 'modules': [], + 'access': [], + 'environs': [ + 'builtin', + 'PrgEnv-cray', + 'PrgEnv-cray_classic', + 'PrgEnv-gnu', + 'PrgEnv-intel', + 'PrgEnv-pgi' + ], + 'descr': 'Login nodes', + 'max_jobs': 4, + 'launcher': 'local' }, - 'PrgEnv-pgi': { + { + 'name': 'gpu', + 'scheduler': 'slurm', + 'container_platforms': [ + { + 'type': 'Sarus', + 'modules': [ + 'sarus' + ] + }, + { + 'type': 'Singularity', + 'modules': [ + 'singularity' + ] + } + ], 'modules': [ - 'PE/17.06', 'pgi/18.5-gcc-5.4.0-2.26', - 'openmpi/4.0.1-pgi-18.5-gcc-5.4.0-2.26-cuda-8.0' + 'daint-gpu' ], - 'cc': 'mpicc', - 'cxx': 'mpicxx', - 'ftn': 'mpifort', - }, - 'PrgEnv-cray': { - 'modules': ['PE/17.06', - 'PrgEnv-CrayCCE/17.06'], - }, - 'PrgEnv-cray-nompi': { - 'modules': ['PE/17.06', - 'PrgEnv-cray'], - }, - 'PrgEnv-gnu': { - 'modules': ['PE/17.06', - 'gmvapich2/17.02_cuda_8.0_gdr'], - 'variables': { - 'LD_PRELOAD': '$(pkg-config --variable=libdir mvapich2-gdr)/libmpi.so' - }, - 'cc': 'mpicc', - 'cxx': 'mpicxx', - 'ftn': 'mpif90', - }, - 'PrgEnv-gnu-nompi': { - 'modules': ['PE/17.06', - 'PrgEnv-gnu'], - 'cc': 'gcc', - 'cxx': 'g++', - 'ftn': 'gfortran', + 'access': [ + '--constraint=gpu' + ], + 'environs': [ + 'builtin', + 'PrgEnv-cray', + 'PrgEnv-cray_classic', + 'PrgEnv-gnu', + 'PrgEnv-intel', + 'PrgEnv-pgi' + ], + 'descr': 'Hybrid nodes (Haswell/P100)', + 'max_jobs': 100, + 'launcher': 'srun', + 'resources': [ + { + 'name': 'gres', + 'options': ['--gres={gres}'] + } + ] }, - }, - - 'arolla': { - 'PrgEnv-pgi-nompi': { - 'type': 'ProgEnvironment', - 'modules': ['PrgEnv-pgi/19.9'], - 'cc': 'pgcc', - 'cxx': 'pgc++', - 'ftn': 'pgf90', + { + 'name': 'mc', + 'scheduler': 'slurm', + 'container_platforms': [ + { + 'type': 'Sarus', + 'modules': [ + 'sarus' + ] + }, + { + 'type': 'Singularity', + 'modules': [ + 'singularity' + ] + } + ], + 'modules': [ + 'daint-mc' + ], + 'access': [ + '--constraint=mc' + ], + 'environs': [ + 'builtin', + 'PrgEnv-cray', + 'PrgEnv-cray_classic', + 'PrgEnv-gnu', + 'PrgEnv-intel', + 'PrgEnv-pgi' + ], + 'descr': 'Multicore nodes (Broadwell)', + 'max_jobs': 100, + 'resources': [ + { + 'name': 'gres', + 'options': ['--gres={gres}'] + } + ], + 'launcher': 'srun' }, - 'PrgEnv-pgi': { - 'type': 'ProgEnvironment', - 'modules': ['PrgEnv-pgi/19.9'], - 'cc': 'mpicc', - 'cxx': 'mpicxx', - 'ftn': 'mpifort', + { + 'name': 'jupyter_gpu', + 'scheduler': 'slurm', + 'environs': [ + 'builtin' + ], + 'access': [ + '-Cgpu', + '--reservation=jupyter_gpu' + ], + 'descr': 'JupyterHub GPU nodes', + 'max_jobs': 10, + 'launcher': 'srun' }, - 'PrgEnv-gnu': { - 'type': 'ProgEnvironment', - 'modules': ['PrgEnv-gnu/19.2'], - 'cc': 'mpicc', - 'cxx': 'mpicxx', - 'ftn': 'mpifort', + { + 'name': 'jupyter_mc', + 'scheduler': 'slurm', + 'environs': [ + 'builtin' + ], + 'access': [ + '-Cmc', + '--reservation=jupyter_mc' + ], + 'descr': 'JupyterHub multicore nodes', + 'max_jobs': 10, + 'launcher': 'srun' + } + ] + }, + { + 'name': 'fulen', + 'descr': 'Fulen', + 'hostnames': [ + r'fulen-ln\d+' + ], + 'modules_system': 'tmod', + 'resourcesdir': '/apps/common/UES/reframe/resources', + 'partitions': [ + { + 'name': 'login', + 'scheduler': 'local', + 'environs': [ + 'PrgEnv-gnu' + ], + 'descr': 'Login nodes', + 'max_jobs': 1, + 'launcher': 'local' }, - 'PrgEnv-gnu-nompi': { - 'type': 'ProgEnvironment', - 'modules': ['PrgEnv-gnu/19.2'], - 'cc': 'gcc', - 'cxx': 'g++', - 'ftn': 'gfortran', + { + 'name': 'normal', + 'scheduler': 'slurm', + 'environs': [ + 'PrgEnv-gnu' + ], + 'descr': 'Compute nodes - default partition', + 'launcher': 'srun' }, - }, - - 'tsa': { - 'PrgEnv-pgi-nompi': { - 'type': 'ProgEnvironment', - 'modules': ['PrgEnv-pgi/19.9'], - 'cc': 'pgcc', - 'cxx': 'pgc++', - 'ftn': 'pgf90', + { + 'name': 'fat', + 'scheduler': 'slurm', + 'environs': [ + 'PrgEnv-gnu' + ], + 'access': [ + '--partition fat' + ], + 'descr': 'High-memory compute nodes', + 'launcher': 'srun' }, - 'PrgEnv-pgi': { - 'type': 'ProgEnvironment', - 'modules': ['PrgEnv-pgi/19.9'], - 'cc': 'mpicc', - 'cxx': 'mpicxx', - 'ftn': 'mpifort', + { + 'name': 'gpu', + 'scheduler': 'slurm', + 'environs': [ + 'PrgEnv-gnu' + ], + 'access': [ + '--partition gpu' + ], + 'descr': 'Hybrid compute nodes', + 'launcher': 'srun' + } + ] + }, + { + 'name': 'kesch', + 'descr': 'Kesch MCH', + 'hostnames': [ + r'keschln-\d+' + ], + 'modules_system': 'tmod', + 'resourcesdir': '/apps/common/UES/reframe/resources', + 'partitions': [ + { + 'name': 'login', + 'scheduler': 'local', + 'environs': [ + 'PrgEnv-cray', + 'PrgEnv-cray-nompi', + 'PrgEnv-pgi', + 'PrgEnv-pgi-nompi', + 'PrgEnv-gnu', + 'PrgEnv-gnu-nompi' + ], + 'descr': 'Kesch login nodes', + 'launcher': 'local' }, - 'PrgEnv-gnu': { - 'type': 'ProgEnvironment', - 'modules': ['PrgEnv-gnu/19.2'], - 'cc': 'mpicc', - 'cxx': 'mpicxx', - 'ftn': 'mpifort', + { + 'name': 'pn', + 'scheduler': 'slurm', + 'access': [ + '--partition=pn-regression' + ], + 'environs': [ + 'PrgEnv-cray', + 'PrgEnv-cray-nompi', + 'PrgEnv-pgi', + 'PrgEnv-pgi-nompi', + 'PrgEnv-gnu', + 'PrgEnv-gnu-nompi' + ], + 'descr': 'Kesch post-processing nodes', + 'launcher': 'srun' }, - 'PrgEnv-gnu-nompi': { - 'type': 'ProgEnvironment', - 'modules': ['PrgEnv-gnu/19.2'], - 'cc': 'gcc', - 'cxx': 'g++', - 'ftn': 'gfortran', + { + 'name': 'cn', + 'scheduler': 'slurm', + 'access': [ + '--partition=cn-regression' + ], + 'environs': [ + 'PrgEnv-cray', + 'PrgEnv-cray-nompi', + 'PrgEnv-pgi', + 'PrgEnv-pgi-nompi', + 'PrgEnv-gnu', + 'PrgEnv-gnu-nompi' + ], + 'descr': 'Kesch compute nodes', + 'resources': [ + { + 'name': '_rfm_gpu', + 'options': [ + '--gres=gpu:{num_gpus_per_node}' + ] + } + ], + 'launcher': 'srun' + } + ] + }, + { + 'name': 'arolla', + 'descr': 'Arolla MCH', + 'hostnames': [ + r'arolla-\w+\d+' + ], + 'modules_system': 'tmod', + 'resourcesdir': '/apps/common/UES/reframe/resources', + 'partitions': [ + { + 'name': 'login', + 'scheduler': 'local', + 'environs': [ + 'PrgEnv-pgi', + 'PrgEnv-pgi-nompi', + 'PrgEnv-gnu', + 'PrgEnv-gnu-nompi' + ], + 'descr': 'Arolla login nodes', + 'launcher': 'local' }, - }, - - '*': { - 'PrgEnv-cray': { - 'modules': ['PrgEnv-cray'], + { + 'name': 'pn', + 'scheduler': 'slurm', + 'access': [ + '--partition=pn-regression' + ], + 'environs': [ + 'PrgEnv-pgi', + 'PrgEnv-pgi-nompi', + 'PrgEnv-gnu', + 'PrgEnv-gnu-nompi' + ], + 'descr': 'Arolla post-processing nodes', + 'launcher': 'srun' }, - - 'PrgEnv-cray_classic': { - 'modules': ['PrgEnv-cray', 'cce/9.0.2-classic'], + { + 'name': 'cn', + 'scheduler': 'slurm', + 'access': [ + '--partition=cn-regression' + ], + 'environs': [ + 'PrgEnv-gnu', + 'PrgEnv-gnu-nompi', + 'PrgEnv-pgi', + 'PrgEnv-pgi-nompi' + ], + 'descr': 'Arolla compute nodes', + 'resources': [ + { + 'name': '_rfm_gpu', + 'options': [ + '--gres=gpu:{num_gpus_per_node}' + ] + } + ], + 'launcher': 'srun' + } + ] + }, + { + 'name': 'tsa', + 'descr': 'Tsa MCH', + 'hostnames': [ + r'tsa-\w+\d+' + ], + 'modules_system': 'tmod', + 'resourcesdir': '/apps/common/UES/reframe/resources', + 'partitions': [ + { + 'name': 'login', + 'scheduler': 'local', + 'environs': [ + 'PrgEnv-pgi', + 'PrgEnv-pgi-nompi', + 'PrgEnv-gnu', + 'PrgEnv-gnu-nompi' + ], + 'descr': 'Tsa login nodes', + 'launcher': 'local' }, - - 'PrgEnv-gnu': { - 'modules': ['PrgEnv-gnu'], + { + 'name': 'pn', + 'scheduler': 'slurm', + 'access': [ + '--partition=pn-regression' + ], + 'environs': [ + 'PrgEnv-pgi', + 'PrgEnv-pgi-nompi', + 'PrgEnv-gnu', + 'PrgEnv-gnu-nompi' + ], + 'descr': 'Tsa post-processing nodes', + 'launcher': 'srun' }, - - 'PrgEnv-intel': { - 'modules': ['PrgEnv-intel'], + { + 'name': 'cn', + 'scheduler': 'slurm', + 'access': [ + '--partition=cn-regression' + ], + 'environs': [ + 'PrgEnv-gnu', + 'PrgEnv-gnu-nompi', + 'PrgEnv-pgi', + 'PrgEnv-pgi-nompi' + ], + 'descr': 'Tsa compute nodes', + 'resources': [ + { + 'name': '_rfm_gpu', + 'options': [ + '--gres=gpu:{num_gpus_per_node}' + ] + } + ], + 'launcher': 'srun' + } + ] + }, + { + 'name': 'generic', + 'descr': 'Generic fallback system', + 'partitions': [ + { + 'name': 'default', + 'scheduler': 'local', + 'environs': [ + 'builtin' + ], + 'descr': 'Login nodes', + 'launcher': 'local' + } + ], + 'hostnames': ['.*'] + } + ], + 'environments': [ + { + 'name': 'PrgEnv-gnu', + 'target_systems': [ + 'ault' + ], + 'modules': [ + 'gcc', + 'cuda/10.1', + 'openmpi' + ], + 'cc': 'mpicc', + 'cxx': 'mpicxx', + 'ftn': 'mpif90' + }, + { + 'name': 'builtin', + 'target_systems': [ + 'ault' + ], + 'cc': 'cc', + 'cxx': '', + 'ftn': '' + }, + { + 'name': 'builtin-gcc', + 'target_systems': [ + 'ault' + ], + 'cc': 'gcc', + 'cxx': 'g++', + 'ftn': 'gfortran' + }, + { + 'name': 'PrgEnv-pgi-nompi', + 'target_systems': [ + 'kesch' + ], + 'modules': [ + 'PE/17.06', + 'PrgEnv-pgi/18.5' + ], + 'cc': 'pgcc', + 'cxx': 'pgc++', + 'ftn': 'pgf90' + }, + { + 'name': 'PrgEnv-pgi', + 'target_systems': [ + 'kesch' + ], + 'modules': [ + 'PE/17.06', + 'pgi/18.5-gcc-5.4.0-2.26', + 'openmpi/4.0.1-pgi-18.5-gcc-5.4.0-2.26-cuda-8.0' + ], + 'cc': 'mpicc', + 'cxx': 'mpicxx', + 'ftn': 'mpifort' + }, + { + 'name': 'PrgEnv-cray', + 'target_systems': [ + 'kesch' + ], + 'modules': [ + 'PE/17.06', + 'PrgEnv-CrayCCE/17.06' + ] + }, + { + 'name': 'PrgEnv-cray-nompi', + 'target_systems': [ + 'kesch' + ], + 'modules': [ + 'PE/17.06', + 'PrgEnv-cray' + ] + }, + { + 'name': 'PrgEnv-gnu', + 'target_systems': [ + 'kesch' + ], + 'modules': [ + 'PE/17.06', + 'gmvapich2/17.02_cuda_8.0_gdr' + ], + 'variables': [ + [ + 'LD_PRELOAD', + '$(pkg-config --variable=libdir mvapich2-gdr)/libmpi.so' + ] + ], + 'cc': 'mpicc', + 'cxx': 'mpicxx', + 'ftn': 'mpif90' + }, + { + 'name': 'PrgEnv-gnu-nompi', + 'target_systems': [ + 'kesch' + ], + 'modules': [ + 'PE/17.06', + 'PrgEnv-gnu' + ], + 'cc': 'gcc', + 'cxx': 'g++', + 'ftn': 'gfortran' + }, + { + 'name': 'PrgEnv-pgi-nompi', + 'target_systems': [ + 'arolla' + ], + 'modules': [ + 'PrgEnv-pgi/19.9' + ], + 'cc': 'pgcc', + 'cxx': 'pgc++', + 'ftn': 'pgf90' + }, + { + 'name': 'PrgEnv-pgi', + 'target_systems': [ + 'arolla' + ], + 'modules': [ + 'PrgEnv-pgi/19.9' + ], + 'cc': 'mpicc', + 'cxx': 'mpicxx', + 'ftn': 'mpifort' + }, + { + 'name': 'PrgEnv-gnu', + 'target_systems': [ + 'arolla' + ], + 'modules': [ + 'PrgEnv-gnu/19.2' + ], + 'cc': 'mpicc', + 'cxx': 'mpicxx', + 'ftn': 'mpifort' + }, + { + 'name': 'PrgEnv-gnu-nompi', + 'target_systems': [ + 'arolla' + ], + 'modules': [ + 'PrgEnv-gnu/19.2' + ], + 'cc': 'gcc', + 'cxx': 'g++', + 'ftn': 'gfortran' + }, + { + 'name': 'PrgEnv-pgi-nompi', + 'target_systems': [ + 'tsa' + ], + 'modules': [ + 'PrgEnv-pgi/19.9' + ], + 'cc': 'pgcc', + 'cxx': 'pgc++', + 'ftn': 'pgf90' + }, + { + 'name': 'PrgEnv-pgi', + 'target_systems': [ + 'tsa' + ], + 'modules': [ + 'PrgEnv-pgi/19.9' + ], + 'cc': 'mpicc', + 'cxx': 'mpicxx', + 'ftn': 'mpifort' + }, + { + 'name': 'PrgEnv-gnu', + 'target_systems': [ + 'tsa' + ], + 'modules': [ + 'PrgEnv-gnu/19.2' + ], + 'cc': 'mpicc', + 'cxx': 'mpicxx', + 'ftn': 'mpifort' + }, + { + 'name': 'PrgEnv-gnu-nompi', + 'target_systems': [ + 'tsa' + ], + 'modules': [ + 'PrgEnv-gnu/19.2' + ], + 'cc': 'gcc', + 'cxx': 'g++', + 'ftn': 'gfortran' + }, + { + 'name': 'PrgEnv-cray', + 'modules': [ + 'PrgEnv-cray' + ] + }, + { + 'name': 'PrgEnv-cray_classic', + 'modules': [ + 'PrgEnv-cray', + 'cce/9.0.2-classic' + ] + }, + { + 'name': 'PrgEnv-gnu', + 'modules': [ + 'PrgEnv-gnu' + ] + }, + { + 'name': 'PrgEnv-intel', + 'modules': [ + 'PrgEnv-intel' + ] + }, + { + 'name': 'PrgEnv-pgi', + 'modules': [ + 'PrgEnv-pgi' + ] + }, + { + 'name': 'builtin', + 'cc': 'cc', + 'cxx': 'CC', + 'ftn': 'ftn' + }, + { + 'name': 'builtin-gcc', + 'cc': 'gcc', + 'cxx': 'g++', + 'ftn': 'gfortran' + } + ], + 'logging': [ + { + 'level': 'debug', + 'handlers': [ + { + 'type': 'file', + 'name': 'reframe.log', + 'level': 'debug', + 'format': '[%(asctime)s] %(levelname)s: %(check_info)s: %(message)s', # noqa: E501 + 'append': False }, - - 'PrgEnv-pgi': { - 'modules': ['PrgEnv-pgi'], + { + 'type': 'stream', + 'name': 'stdout', + 'level': 'info', + 'format': '%(message)s' }, - - 'builtin': { - 'cc': 'cc', - 'cxx': 'CC', - 'ftn': 'ftn', + { + 'type': 'file', + 'name': 'reframe.out', + 'level': 'info', + 'format': '%(message)s', + 'append': False + } + ], + 'handlers_perflog': [ + { + 'type': 'filelog', + 'prefix': '%(check_system)s/%(check_partition)s', + 'level': 'info', + 'format': '%(check_job_completion_time)s|reframe %(version)s|%(check_info)s|jobid=%(check_jobid)s|num_tasks=%(check_num_tasks)s|%(check_perf_var)s=%(check_perf_value)s|ref=%(check_perf_ref)s (l=%(check_perf_lower_thres)s, u=%(check_perf_upper_thres)s)|%(check_perf_unit)s', # noqa: E501 + 'datefmt': '%FT%T%:z', + 'append': True }, - - 'builtin-gcc': { - 'cc': 'gcc', - 'cxx': 'g++', - 'ftn': 'gfortran', + { + 'type': 'graylog', + 'address': 'graylog-server:12345', + 'level': 'info', + 'format': '%(message)s', + 'extras': { + 'facility': 'reframe', + 'data-version': '1.0', + } } - } + ] + } + ], + 'modes': [ + { + 'name': 'maintenance', + 'options': [ + '--unload-module=reframe', + '--exec-policy=async', + '--strict', + '--output=$APPS/UES/$USER/regression/maintenance', + '--perflogdir=$APPS/UES/$USER/regression/maintenance/logs', + '--stage=$SCRATCH/regression/maintenance/stage', + '--reservation=maintenance', + '--save-log-files', + '--tag=maintenance', + '--timestamp=%F_%H-%M-%S' + ] }, - - 'modes': { - '*': { - 'maintenance': [ - '--unload-module=reframe', - '--exec-policy=async', - '--strict', - '--output=$APPS/UES/$USER/regression/maintenance', - '--perflogdir=$APPS/UES/$USER/regression/maintenance/logs', - '--stage=$SCRATCH/regression/maintenance/stage', - '--reservation=maintenance', - '--save-log-files', - '--tag=maintenance', - '--timestamp=%F_%H-%M-%S' - ], - 'production': [ - '--unload-module=reframe', - '--exec-policy=async', - '--strict', - '--output=$APPS/UES/$USER/regression/production', - '--perflogdir=$APPS/UES/$USER/regression/production/logs', - '--stage=$SCRATCH/regression/production/stage', - '--save-log-files', - '--tag=production', - '--timestamp=%F_%H-%M-%S' - ] - } + { + 'name': 'production', + 'options': [ + '--unload-module=reframe', + '--exec-policy=async', + '--strict', + '--output=$APPS/UES/$USER/regression/production', + '--perflogdir=$APPS/UES/$USER/regression/production/logs', + '--stage=$SCRATCH/regression/production/stage', + '--save-log-files', + '--tag=production', + '--timestamp=%F_%H-%M-%S' + ] } - } - - logging_config = { - 'level': 'DEBUG', - 'handlers': [ - { - 'type': 'file', - 'name': 'reframe.log', - 'level': 'DEBUG', - 'format': '[%(asctime)s] %(levelname)s: ' - '%(check_info)s: %(message)s', - 'append': False, - }, - - # Output handling - { - 'type': 'stream', - 'name': 'stdout', - 'level': 'INFO', - 'format': '%(message)s' - }, - { - 'type': 'file', - 'name': 'reframe.out', - 'level': 'INFO', - 'format': '%(message)s', - 'append': False, - } - ] - } - - perf_logging_config = { - 'level': 'DEBUG', - 'handlers': [ - #@ { - #@ 'type': 'graylog', - #@ 'host': 'your-server-here', - #@ 'port': 12345, - #@ 'level': 'INFO', - #@ 'format': '%(message)s', - #@ 'extras': { - #@ 'facility': 'reframe', - #@ 'data-version': '1.0', - #@ } - #@ }, - { - 'type': 'filelog', - 'prefix': '%(check_system)s/%(check_partition)s', - 'level': 'INFO', - 'format': ( - '%(check_job_completion_time)s|reframe %(version)s|' - '%(check_info)s|jobid=%(check_jobid)s|' - 'num_tasks=%(check_num_tasks)s|' - '%(check_perf_var)s=%(check_perf_value)s|' - 'ref=%(check_perf_ref)s ' - '(l=%(check_perf_lower_thres)s, ' - 'u=%(check_perf_upper_thres)s)|' - '%(check_perf_unit)s' - ), - 'datefmt': '%FT%T%:z', - 'append': True - } - ] - } - - -settings = ReframeSettings() + ], + 'general': [ + { + 'check_search_path': [ + 'checks/' + ], + 'check_search_recursive': True + } + ] +} diff --git a/config/tiger.py b/config/tiger.py index 9c56748048..2dda062b72 100644 --- a/config/tiger.py +++ b/config/tiger.py @@ -4,159 +4,177 @@ # SPDX-License-Identifier: BSD-3-Clause # -# CSCS ReFrame settings +# ReFrame settings for Cray Tiger # - -class ReframeSettings: - job_poll_intervals = [1, 2, 3] - job_submit_timeout = 60 - checks_path = ['checks/'] - checks_path_recurse = True - site_configuration = { - 'systems': { - 'tiger': { - # export SCRATCH=/lus/scratch/$USER - 'descr': 'Cray Tiger', - 'hostnames': ['tiger'], - 'modules_system': 'tmod', - 'resourcesdir': '$HOME/RESOURCES', - 'partitions': { - 'login': { - 'scheduler': 'local', - 'modules': [], - 'access': [], - 'environs': ['PrgEnv-cray', 'PrgEnv-gnu', - 'PrgEnv-intel', 'PrgEnv-pgi'], - 'descr': 'Login nodes', - 'max_jobs': 4 - }, - - 'gpu': { - 'scheduler': 'nativeslurm', - 'modules': ['craype-broadwell'], - 'access': ['--constraint=P100'], - 'environs': ['PrgEnv-cray', 'PrgEnv-gnu', - 'PrgEnv-intel', 'PrgEnv-pgi'], - 'descr': 'Hybrid nodes (Broadwell/P100)', - 'max_jobs': 100, - 'resources': { - 'switches': ['--switches={num_switches}'], - 'mem-per-cpu': ['--mem-per-cpu={mem_per_cpu}'] +site_configuration = { + 'systems': [ + { + 'name': 'tiger', + 'descr': 'Cray Tiger', + 'hostnames': [ + 'tiger' + ], + 'modules_system': 'tmod', + 'resourcesdir': '$HOME/RESOURCES', + 'partitions': [ + { + 'name': 'login', + 'scheduler': 'local', + 'modules': [], + 'access': [], + 'environs': [ + 'PrgEnv-cray', + 'PrgEnv-gnu', + 'PrgEnv-intel', + 'PrgEnv-pgi' + ], + 'descr': 'Login nodes', + 'max_jobs': 4, + 'launcher': 'local' + }, + { + 'name': 'gpu', + 'scheduler': 'slurm', + 'modules': [ + 'craype-broadwell' + ], + 'access': [ + '--constraint=P100' + ], + 'environs': [ + 'PrgEnv-cray', + 'PrgEnv-gnu', + 'PrgEnv-intel', + 'PrgEnv-pgi' + ], + 'descr': 'Hybrid nodes (Broadwell/P100)', + 'max_jobs': 100, + 'resources': [ + { + 'name': 'switches', + 'options': [ + '--switches={num_switches}' + ] + }, + { + 'name': 'mem-per-cpu', + 'options': [ + '--mem-per-cpu={mem_per_cpu}' + ] } - } + ], + 'launcher': 'srun' } - }, - - 'generic': { - 'descr': 'Generic example system', - 'partitions': { - 'login': { - 'scheduler': 'local', - 'modules': [], - 'access': [], - 'environs': ['builtin-gcc'], - 'descr': 'Login nodes' - } + ] + }, + { + 'name': 'generic', + 'descr': 'Generic example system', + 'partitions': [ + { + 'name': 'login', + 'scheduler': 'local', + 'modules': [], + 'access': [], + 'environs': [ + 'builtin-gcc' + ], + 'descr': 'Login nodes', + 'launcher': 'local' } - } + ], + 'hostnames': [r'.*'] + } + ], + 'environments': [ + { + 'name': 'PrgEnv-cray', + 'modules': [ + 'PrgEnv-cray' + ] }, - - 'environments': { - '*': { - 'PrgEnv-cray': { - 'type': 'ProgEnvironment', - 'modules': ['PrgEnv-cray'], - }, - - 'PrgEnv-cray_classic': { - 'type': 'ProgEnvironment', - 'modules': ['PrgEnv-cray', 'cce/9.1.0-classic'], - }, - - 'PrgEnv-gnu': { - 'type': 'ProgEnvironment', - 'modules': ['PrgEnv-gnu'], - }, - - 'PrgEnv-intel': { - 'type': 'ProgEnvironment', - 'modules': ['PrgEnv-intel'], - }, - - 'PrgEnv-pgi': { - 'type': 'ProgEnvironment', - 'modules': ['PrgEnv-pgi'], + { + 'name': 'PrgEnv-cray_classic', + 'modules': [ + 'PrgEnv-cray', + 'cce/9.1.0-classic' + ] + }, + { + 'name': 'PrgEnv-gnu', + 'modules': [ + 'PrgEnv-gnu' + ] + }, + { + 'name': 'PrgEnv-intel', + 'modules': [ + 'PrgEnv-intel' + ] + }, + { + 'name': 'PrgEnv-pgi', + 'modules': [ + 'PrgEnv-pgi' + ] + }, + { + 'name': 'builtin', + 'cc': 'cc', + 'cxx': '', + 'ftn': '' + }, + { + 'name': 'builtin-gcc', + 'cc': 'gcc', + 'cxx': 'g++', + 'ftn': 'gfortran' + } + ], + 'logging': [ + { + 'level': 'debug', + 'handlers': [ + { + 'type': 'file', + 'name': 'reframe.log', + 'level': 'debug', + 'format': '[%(asctime)s] %(levelname)s: %(check_info)s: %(message)s', # noqa: E501 + 'append': False }, - - 'builtin': { - 'type': 'ProgEnvironment', - 'cc': 'cc', - 'cxx': '', - 'ftn': '', + { + 'type': 'stream', + 'name': 'stdout', + 'level': 'info', + 'format': '%(message)s' }, - - 'builtin-gcc': { - 'type': 'ProgEnvironment', - 'cc': 'gcc', - 'cxx': 'g++', - 'ftn': 'gfortran', + { + 'type': 'file', + 'name': 'reframe.out', + 'level': 'info', + 'format': '%(message)s', + 'append': False } - } - }, - - } - - logging_config = { - 'level': 'DEBUG', - 'handlers': [ - { - 'type': 'file', - 'name': 'reframe.log', - 'level': 'DEBUG', - 'format': '[%(asctime)s] %(levelname)s: ' - '%(check_info)s: %(message)s', - 'append': False, - }, - - # Output handling - { - 'type': 'stream', - 'name': 'stdout', - 'level': 'INFO', - 'format': '%(message)s' - }, - { - 'type': 'file', - 'name': 'reframe.out', - 'level': 'INFO', - 'format': '%(message)s', - 'append': False, - } - ] - } - - perf_logging_config = { - 'level': 'DEBUG', - 'handlers': [ - { - 'type': 'filelog', - 'prefix': '%(check_system)s/%(check_partition)s', - 'level': 'INFO', - 'format': ( - '%(check_job_completion_time)s|reframe %(version)s|' - '%(check_info)s|jobid=%(check_jobid)s|' - '%(check_perf_var)s=%(check_perf_value)s|' - 'ref=%(check_perf_ref)s ' - '(l=%(check_perf_lower_thres)s, ' - 'u=%(check_perf_upper_thres)s)|' - '%(check_perf_unit)s' - ), - 'datefmt': '%FT%T%:z', - 'append': True - } - ] - } - - -settings = ReframeSettings() + ], + 'handlers_perflog': [ + { + 'type': 'filelog', + 'prefix': '%(check_system)s/%(check_partition)s', + 'level': 'info', + 'format': '%(check_job_completion_time)s|reframe %(version)s|%(check_info)s|jobid=%(check_jobid)s|%(check_perf_var)s=%(check_perf_value)s|ref=%(check_perf_ref)s (l=%(check_perf_lower_thres)s, u=%(check_perf_upper_thres)s)|%(check_perf_unit)s', # noqa: E501 + 'datefmt': '%FT%T%:z', + 'append': True + } + ] + } + ], + 'general': [ + { + 'check_search_path': [ + 'checks/' + ], + 'check_search_recursive': True + } + ] +} diff --git a/cscs-checks/analytics/spark/spark_check.py b/cscs-checks/analytics/spark/spark_check.py index b1e3e2e530..52bfa0981c 100644 --- a/cscs-checks/analytics/spark/spark_check.py +++ b/cscs-checks/analytics/spark/spark_check.py @@ -8,8 +8,7 @@ import reframe as rfm import reframe.utility.sanity as sn -from reframe.core.launchers import LauncherWrapper -from reframe.core.launchers.registry import getlauncher +from reframe.core.backends import getlauncher @rfm.simple_test diff --git a/cscs-checks/apps/spark/spark_check.py b/cscs-checks/apps/spark/spark_check.py index df95ea1bc0..bc8733f84b 100644 --- a/cscs-checks/apps/spark/spark_check.py +++ b/cscs-checks/apps/spark/spark_check.py @@ -7,7 +7,7 @@ import reframe as rfm import reframe.utility.sanity as sn -from reframe.core.launchers.registry import getlauncher +from reframe.core.backends import getlauncher @rfm.simple_test diff --git a/cscs-checks/microbenchmarks/spec-accel/spec.py b/cscs-checks/microbenchmarks/spec-accel/spec.py index 4cf1994020..51320501fb 100644 --- a/cscs-checks/microbenchmarks/spec-accel/spec.py +++ b/cscs-checks/microbenchmarks/spec-accel/spec.py @@ -7,7 +7,7 @@ import reframe as rfm import reframe.utility.sanity as sn -from reframe.core.launchers.registry import getlauncher +from reframe.core.backends import getlauncher class SpecAccelCheckBase(rfm.RegressionTest): diff --git a/docs/_static/img/gromacs-perf.png b/docs/_static/img/gromacs-perf.png new file mode 100644 index 0000000000..4bc6df2725 Binary files /dev/null and b/docs/_static/img/gromacs-perf.png differ diff --git a/docs/config_reference.rst b/docs/config_reference.rst new file mode 100644 index 0000000000..b00e9f0e6b --- /dev/null +++ b/docs/config_reference.rst @@ -0,0 +1,1215 @@ +======================= +Configuration Reference +======================= + +.. versionadded:: 3.0 + + +ReFrame's behavior can be configured through its configuration file (see `Configuring ReFrame for Your Site `__), environment variables and command-line options. +An option can be specified via multiple paths (e.g., a configuration file parameter and an environment variable), in which case command-line options precede environment variables, which in turn precede configuration file options. +This section provides a complete reference guide of the configuration options of ReFrame that can be set in its configuration file or specified using environment variables. + +ReFrame's configuration is in JSON syntax. +The full schema describing it can be found in `schemas/config.json `__ file. +Any configuration file given to ReFrame is validated against this schema. + +The syntax we use in the following to describe the different configuration object attributes is a valid query string for the `jq `__ JSON command-line processor. +Along the configuration option, the corresponding environment variable and command-line options are listed, if any. + + +Top-level Configuration +----------------------- + +The top-level configuration object is essentially the full configuration of ReFrame. +It consists of the following properties: + +.. py:attribute:: .systems + + :required: Yes + + A list of `system configuration objects <#system-configuration>`__. + + +.. py:attribute:: .environments + + :required: Yes + + A list of `environment configuration objects <#environment-configuration>`__. + + +.. py:attribute:: .logging + + :required: Yes + + A list of `logging configuration objects <#logging-configuration>`__. + + +.. py:attribute:: .schedulers + + :required: No + + A list of `scheduler configuration objects <#scheduler-configuration>`__. + + +.. py:attribute:: .modes + + :required: No + + A list of `execution mode configuration objects <#execution-mode-configuration>`__. + +.. py:attribute:: .general + + :required: No + + A list of `general configuration objects <#general-configuration>`__. + + +System Configuration +-------------------- + +.. js:attribute:: .systems[].name + + :required: Yes + + The name of this system. + Only alphanumeric characters, dashes (``-``) and underscores (``_``) are allowed. + +.. js:attribute:: .systems[].descr + + :required: No + :default: ``""`` + + The description of this system. + +.. js:attribute:: .systems[].hostnames + + :required: Yes + + A list of hostname regular expression patterns in Python `syntax `__, which will be used by the framework in order to automatically select a system configuration. + For the auto-selection process, see `here `__. + +.. js:attribute:: .systems[].modules_system + + :required: No + :default: ``"nomod"`` + + The modules system that should be used for loading environment modules on this system. + Available values are the following: + + - ``tmod``: The classic Tcl implementation of the `environment modules `__ (version 3.2). + - ``tmod31``: The classic Tcl implementation of the `environment modules `__ (version 3.1). + A separate backend is required for Tmod 3.1, because Python bindings are different from Tmod 3.2. + - ``tmod32``: A synonym of ``tmod``. + - ``tmod4``: The `new environment modules `__ implementation (versions older than 4.1 are not supported). + - ``lmod``: The `Lua implementation `__ of the environment modules. + - ``nomod``: This is to denote that no modules system is used by this system. + +.. js:attribute:: .systems[].modules + + :required: No + :default: ``[]`` + + Environment modules to be loaded always when running on this system. + These modules modify the ReFrame environment. + This is useful in cases where a particular module is needed, for example, to submit jobs on a specific system. + +.. js:attribute:: .systems[].variables + + :required: No + :default: ``[]`` + + A list of environment variables to be set always when running on this system. + Each environment variable is specified as a two-element list containing the variable name and its value. + You may reference other environment variables when defining an environment variable here. + ReFrame will expand its value. + Variables are set after the environment modules are loaded. + +.. js:attribute:: .systems[].prefix + +.. envvar:: RFM_PREFIX + +.. option:: --prefix + + :required: No + :default: ``"."`` + + Directory prefix for a ReFrame run on this system. + Any directories or files produced by ReFrame will use this prefix, if not specified otherwise. + +.. js:attribute:: .systems[].stagedir + +.. envvar:: RFM_STAGE_DIR + +.. option:: -s DIR | --stage DIR + + :required: No + :default: ``"${RFM_PREFIX}/stage"`` + + Stage directory prefix for this system. + This is the directory prefix, where ReFrame will create the stage directories for each individual test case. + + +.. js:attribute:: .systems[].outputdir + +.. envvar:: RFM_OUTPUT_DIR + +.. option:: -o DIR | --output DIR + + :required: No + :default: ``"${RFM_PREFIX}/output"`` + + Output directory prefix for this system. + This is the directory prefix, where ReFrame will save information about the successful tests. + + +.. js:attribute:: .systems[].resourcesdir + + :required: No + :default: ``"."`` + + Directory prefix where external test resources (e.g., large input files) are stored. + You may reference this prefix from within a regression test by accessing the corresponding `attribute `__ of the current system. + + +.. js:attribute:: .systems[].partitions + + :required: Yes + + A list of `system partition configuration objects <#system-partition-configuration>`__. + This list must have at least one element. + + +------------------------------ +System Partition Configuration +------------------------------ + +.. js:attribute:: .systems[].partitions[].name + + :required: Yes + + The name of this partition. + Only alphanumeric characters, dashes (``-``) and underscores (``_``) are allowed. + +.. js:attribute:: .systems[].partitions[].descr + + :required: No + :default: ``""`` + + The description of this partition. + +.. js:attribute:: .systems[].partitions[].scheduler + + :required: Yes + + The job scheduler that will be used to launch jobs on this partition. + Supported schedulers are the following: + + - ``local``: Jobs will be launched locally without using any job scheduler. + - ``pbs``: Jobs will be launched using the `PBS Pro `__ scheduler. + - ``torque``: Jobs will be launched using the `Torque `__ scheduler. + - ``slurm``: Jobs will be launched using the `Slurm `__ scheduler. + This backend requires job accounting to be enabled in the target system. + If not, you should consider using the ``squeue`` backend below. + - ``squeue``: Jobs will be launched using the `Slurm `__ scheduler. + This backend does not rely on job accounting to retrieve job statuses, but ReFrame does its best to query the job state as reliably as possible. + +.. js:attribute:: .systems[].partitions[].launcher + + :required: Yes + + The parallel job launcher that will be used in this partition to launch parallel programs. + Available values are the following: + + - ``alps``: Parallel programs will be launched using the `Cray ALPS `__ ``aprun`` command. + - ``ibrun``: Parallel programs will be launched using the ``ibrun`` command. + This is a custom parallel program launcher used at `TACC `__. + - ``local``: No parallel program launcher will be used. + The program will be launched locally. + - ``mpirun``: Parallel programs will be launched using the ``mpirun`` command. + - ``mpiexec``: Parallel programs will be launched using the ``mpiexec`` command. + - ``srun``: Parallel programs will be launched using `Slurm `__'s ``srun`` command. + - ``srunalloc``: Parallel programs will be launched using `Slurm `__'s ``srun`` command, but job allocation options will also be emitted. + This can be useful when combined with the ``local`` job scheduler. + - ``ssh``: Parallel programs will be launched using SSH. + This launcher uses the partition’s `access `__ property in order to determine the remote host and any additional options to be passed to the SSH client. + The ssh command will be launched in "batch mode," meaning that password-less access to the remote host must be configured. + Here is an example configuration for the ssh launcher: + + .. code:: python + + { + 'name': 'foo' + 'scheduler': 'local', + 'launcher': 'ssh' + 'access': ['-l admin', 'remote.host'], + 'environs': ['builtin'], + } + +.. js:attribute:: .systems[].partitions[].access + + :required: No + :default: ``[]`` + + A list of job scheduler options that will be passed to the generated job script for gaining access to that logical partition. + + +.. js:attribute:: .systems[].partitions[].environs + + :required: No + :default: ``[]`` + + A list of environment names that ReFrame will use to run regression tests on this partition. + Each environment must be defined in the `environments `__ section of the configuration and the definition of the environment must be valid for this partition. + + +.. js:attribute:: .systems[].partitions[].container_platforms + + :required: No + :default: ``[]`` + + A list for `container platform configuration objects <#container-platform-configuration>`__. + This will allow launching regression tests that use `containers `__ on this partition. + + +.. js:attribute:: .systems[].partitions[].modules + + :required: No + :default: ``[]`` + + A list of environment modules to be loaded before running a regression test on this partition. + + +.. js:attribute:: .systems[].partitions[].variables + + :required: No + :default: ``[]`` + + A list of environment variables to be set before running a regression test on this partition. + Each environment variable is specified as a two-element list containing the variable name and its value. + You may reference other environment variables when defining an environment variable here. + ReFrame will expand its value. + Variables are set after the environment modules are loaded. + + +.. js:attribute:: .systems[].partitions[].max_jobs + + :required: No + :default: ``1`` + + The maximum number of concurrent regression tests that may be active (i.e., not completed) on this partition. + This option is relevant only when ReFrame executes with the `asynchronous execution policy `__. + + +.. js:attribute:: .systems[].partitions[].resources + + :required: No + :default: ``[]`` + + A list of job scheduler `resource specification <#config_reference.html#custom-job-scheduler-resources>`__ objects. + + + +Container Platform Configuration +================================ + +ReFrame can launch containerized applications, but you need to configure properly a system partition in order to do that by defining a container platform configuration. + +.. js:attribute:: .systems[].partitions[].container_platforms[].type + + :required: Yes + + The type of the container platform. + Available values are the following: + + - ``Docker``: The `Docker `__ container runtime. + - ``Sarus``: The `Sarus `__ container runtime. + - ``Shifter``: The `Shifter `__ container runtime. + - ``Singularity``: The `Singularity `__ container runtime. + + +.. js:attribute:: .systems[].partitions[].container_platforms[].modules + + :required: No + :default: ``[]`` + + List of environment modules to be loaded when running containerized tests using this container platform. + + +.. js:attribute:: .systems[].partitions[].container_platforms[].variables + + :required: No + :default: ``[]`` + + List of environment variables to be set when running containerized tests using this container platform. + Each environment variable is specified as a two-element list containing the variable name and its value. + You may reference other environment variables when defining an environment variable here. + ReFrame will expand its value. + Variables are set after the environment modules are loaded. + + +Custom Job Scheduler Resources +============================== + +ReFrame allows you to define custom scheduler resources for each partition that you can then transparently access through the :attr:`extra_resources` attribute of a regression test. + +.. js:attribute:: .systems[].partitions[].resources[].name + + :required: Yes + + The name of this resources. + This name will be used to request this resource in a regression test's :attr:`extra_resources`. + + +.. js:attribute:: .systems[].partitions[].resources[].options + + :required: No + :default: ``[]`` + + A list of options to be passed to this partition’s job scheduler. + The option strings can contain placeholders of the form ``{placeholder_name}``. + These placeholders may be replaced with concrete values by a regression test through the :attr:`extra_resources` attribute. + + For example, one could define a ``gpu`` resource for a multi-GPU system that uses Slurm as follows: + + .. code:: python + + 'resources': [ + { + 'name': 'gpu', + 'options': ['--gres=gpu:{num_gpus_per_node}'] + } + ] + + + A regression test then may request this resource as follows: + + .. code:: python + + self.extra_resources = {'gpu': {'num_gpus_per_node': '8'}} + + + And the generated job script will have the following line in its preamble: + + .. code:: bash + + #SBATCH --gres=gpu:8 + + + A resource specification may also start with ``#PREFIX``, in which case ``#PREFIX`` will replace the standard job script prefix of the backend scheduler of this partition. + This is useful in cases of job schedulers like Slurm, that allow alternative prefixes for certain features. + An example is the `DataWarp `__ functionality of Slurm which is supported by the ``#DW`` prefix. + One could then define DataWarp related resources as follows: + + .. code:: python + + 'resources': [ + { + 'name': 'datawarp', + 'options': [ + '#DW jobdw capacity={capacity} access_mode={mode} type=scratch', + '#DW stage_out source={out_src} destination={out_dst} type={stage_filetype}' + ] + } + ] + + + A regression test that wants to make use of that resource, it can set its :attr:`extra_resources` as follows: + + .. code:: python + + self.extra_resources = { + 'datawarp': { + 'capacity': '100GB', + 'mode': 'striped', + 'out_src': '$DW_JOB_STRIPED/name', + 'out_dst': '/my/file', + 'stage_filetype': 'file' + } + } + + .. note:: + + For the ``pbs`` and ``torque`` backends, options accepted in the `access <#.systems[].partitions[].access>`__ and `resources <#.systems[].partitions[].resources>`__ attributes may either refer to actual ``qsub`` options or may be just resources specifications to be passed to the ``-l`` option. + The backend assumes a ``qsub`` option, if the options passed in these attributes start with a ``-``. + + +Environment Configuration +------------------------- + +Environments defined in this section will be used for running regression tests. +They are associated with `system partitions <#system-partition-configuration>`__. + + +.. js:attribute:: .environments[].name + + :required: Yes + + The name of this environment. + + +.. js:attribute:: .environments[].modules + + :required: No + :default: ``[]`` + + A list of environment modules to be loaded when this environment is loaded. + + +.. js:attribute:: .environments[].variables + + :required: No + :default: ``[]`` + + A list of environment variables to be set when loading this environment. + Each environment variable is specified as a two-element list containing the variable name and its value. + You may reference other environment variables when defining an environment variable here. + ReFrame will expand its value. + Variables are set after the environment modules are loaded. + + +.. js:attribute:: .environments[].cc + + :required: No + :default: ``"cc"`` + + The C compiler to be used with this environment. + + +.. js:attribute:: .environments[].cxx + + :required: No + :default: ``"CC"`` + + The C++ compiler to be used with this environment. + + +.. js:attribute:: .environments[].ftn + + :required: No + :default: ``"ftn"`` + + The Fortran compiler to be used with this environment. + + +.. js:attribute:: .environments[].cppflags + + :required: No + :default: ``[]`` + + A list of C preprocessor flags to be used with this environment by default. + + +.. js:attribute:: .environments[].cflags + + :required: No + :default: ``[]`` + + A list of C flags to be used with this environment by default. + + +.. js:attribute:: .environments[].cxxflags + + :required: No + :default: ``[]`` + + A list of C++ flags to be used with this environment by default. + + +.. js:attribute:: .environments[].fflags + + :required: No + :default: ``[]`` + + A list of Fortran flags to be used with this environment by default. + + +.. js:attribute:: .environments[].ldflags + + :required: No + :default: ``[]`` + + A list of linker flags to be used with this environment by default. + + +.. js:attribute:: .environments[].target_systems + + :required: No + :default: ``["*"]`` + + A list of systems or system/partitions combinations that this environment definition is valid for. + A ``*`` entry denotes any system. + In case of multiple definitions of an environment, the most specific to the current system partition will be used. + For example, if the current system/partition combination is ``daint:mc``, the second definition of the ``PrgEnv-gnu`` environment will be used: + + .. code:: python + + 'environments': [ + { + 'name': 'PrgEnv-gnu', + 'modules': ['PrgEnv-gnu'] + }, + { + 'name': 'PrgEnv-gnu', + 'modules': ['PrgEnv-gnu', 'openmpi'], + 'cc': 'mpicc', + 'cxx': 'mpicxx', + 'ftn': 'mpif90', + 'target_systems': ['daint:mc'] + } + ] + + However, if the current system was ``daint:gpu``, the first definition would be selected, despite the fact that the second definition is relevant for another partition of the same system. + To better understand this, ReFrame resolves definitions in a hierarchical way. + It first looks for definitions for the current partition, then for the containing system and, finally, for global definitions (the ``*`` pseudo-system). + + +Logging Configuration +--------------------- + +Logging in ReFrame is handled by logger objects which further delegate message to *logging handlers* which are eventually responsible for emitting or sending the log records to their destinations. +You may define different logger objects per system but *not* per partition. + + +.. js:attribute:: .logging[].level + + :required: No + :default: ``"debug"`` + + The level associated with this logger object. + There are the following levels in decreasing severity order: + + - ``critical``: Catastrophic errors; the framework cannot proceed with its execution. + - ``error``: Normal errors; the framework may or may not proceed with its execution. + - ``warning``: Warning messages. + - ``info``: Informational messages. + - ``verbose``: More informational messages. + - ``debug``: Debug messages. + + If a message is logged by the framework, its severity level will be checked by the logger and if it is higher from the logger's level, it will be passed down to its handlers. + + +.. js:attribute:: .logging[].handlers + + :required: Yes + + A list of `logging handlers <#logging-handlers>`__ responsible for handling normal framework output. + + +.. js:attribute:: .logging[].handlers_perflog + + :required: Yes + + A list of logging handlers responsible for handling `performance data <#performance-logging-handlers>`__ from tests. + + +.. js:attribute:: .logging[].target_systems + + :required: No + :default: ``["*"]`` + + A list of systems or system/partitions combinations that this logging configuration is valid for. + For a detailed description of this property, you may refer `here <#.environments[].target_systems>`__. + + + +--------------------------------- +Common logging handler properties +--------------------------------- + +All logging handlers share the following set of common attributes: + + +.. js:attribute:: .logging[].handlers[].type + +.. js:attribute:: .logging[].handlers_perflog[].type + + :required: Yes + + The type of handler. + There are the following types available: + + - ``file``: This handler sends log records to file. + See `here <#the-file-handler>`__ for more details. + - ``filelog``: This handler sends performance log records to files. + See `here <#the-filelog-handler>`__ for more details. + - ``graylog``: This handler sends performance log records to Graylog. + See `here <#the-graylog-handler>`__ for more details. + - ``stream``: This handler sends log records to a file stream. + See `here <#the-stream-handler>`__ for more details. + - ``syslog``: This handler sends log records to a Syslog facility. + See `here <#the-syslog-handler>`__ for more details. + + +.. js:attribute:: .logging[].handlers[].level + +.. js:attribute:: .logging[].handlers_perflog[].level + + :required: No + :default: ``"info"`` + + The `log level <#.logging[].level>`__ associated with this handler. + + + +.. js:attribute:: .logging[].handlers[].format + +.. js:attribute:: .logging[].handlers_perflog[].format + + :required: No + :default: ``"%(message)s"`` + + Log record format string. + ReFrame accepts all log record attributes from Python's `logging `__ mechanism and adds the following: + + - ``%(check_environ)s``: The name of the `environment <#environment-configuration>`__ that the current test is being executing for. + - ``%(check_info)s``: General information of the currently executing check. + By default this field has the form ``%(check_name)s on %(check_system)s:%(check_partition)s using %(check_environ)s``. + It can be configured on a per test basis by overriding the :func:`info ` method of a specific regression test. + - ``%(check_jobid)s``: The job or process id of the job or process associated with the currently executing regression test. + If a job or process is not yet created, ``-1`` will be printed. + - ``%(check_job_completion_time)s``: The completion time of the job spawned by this regression test. + This timestamp will be formatted according to `datefmt <#.logging[].handlers[].datefmt>`__ handler property. + The accuracy of this timestamp depends on the backend scheduler. + The ``slurm`` scheduler `backend <#.systems[].partitions[].scheduler>`__ relies on job accounting and returns the actual termination time of the job. + The rest of the backends report as completion time the moment when the framework realizes that the spawned job has finished. + In this case, the accuracy depends on the execution policy used. + If tests are executed with the serial execution policy, this is close to the real completion time, but if the asynchronous execution policy is used, it can differ significantly. + If the job completion time cannot be retrieved, ``None`` will be printed. + - ``%(check_job_completion_time_unix)s``: The completion time of the job spawned by this regression test expressed as UNIX time. + This is a raw time field and will not be formatted according to ``datefmt``. + If specific formatting is desired, the ``check_job_completion_time`` should be used instead. + - ``%(check_name)s``: The name of the regression test on behalf of which ReFrame is currently executing. + If ReFrame is not executing in the context of a regression test, ``reframe`` will be printed instead. + - ``%(check_num_tasks)s``: The number of tasks assigned to the regression test. + - ``%(check_outputdir)s``: The output directory associated with the currently executing test. + - ``%(check_partition)s``: The system partition where this test is currently executing. + - ``%(check_stagedir)s``: The stage directory associated with the currently executing test. + - ``%(check_system)s``: The system where this test is currently executing. + - ``%(check_tags)s``: The tags associated with this test. + - ``%(check_perf_lower_thres)s``: The lower threshold of the performance difference from the reference value expressed as a fractional value. + See the `reference `__ attribute of regression tests for more details. + - ``%(check_perf_ref)s``: The reference performance value of a certain performance variable. + - ``%(check_perf_unit)s``: The unit of measurement for the measured performance variable. + - ``%(check_perf_upper_thres)s``: The lower threshold of the performance difference from the reference value expressed as a fractional value. + See the `reference `__ attribute of regression tests for more details. + - ``%(check_perf_value)s``: The performance value obtained for a certain performance variable. + - ``%(check_perf_var)s``: The name of the `performance variable `__ being logged. + - ``%(osuser)s``: The name of the OS user running ReFrame. + - ``%(osgroup)s``: The name of the OS group running ReFrame. + - ``%(version)s``: The ReFrame version. + + +.. js:attribute:: .logging[].handlers[].datefmt + +.. js:attribute:: .logging[].handlers_perflog[].datefmt + + :required: No + :default: ``"%FT%T"`` + + Time format to be used for printing timestamps fields. + There are two timestamp fields available: ``%(asctime)s`` and ``%(check_job_completion_time)s``. + In addition to the format directives supported by the standard library's `time.strftime() `__ function, ReFrame allows you to use the ``%:z`` directive -- a GNU ``date`` extension -- that will print the time zone difference in a RFC3339 compliant way, i.e., ``+/-HH:MM`` instead of ``+/-HHMM``. + + +------------------------ +The ``file`` log handler +------------------------ + +This log handler handles output to normal files. +The additional properties for the ``file`` handler are the following: + + +.. js:attribute:: .logging[].handlers[].name + +.. js:attribute:: .logging[].handlers_perflog[].name + + :required: Yes + + The name of the file where this handler will write log records. + + +.. js:attribute:: .logging[].handlers[].append + +.. js:attribute:: .logging[].handlers_perflog[].append + + :required: No + :default: ``false`` + + Controls whether this handler should append to its file or not. + + +.. js:attribute:: .logging[].handlers[].timestamp + +.. js:attribute:: .logging[].handlers_perflog[].timestamp + + :required: No + :default: ``false`` + + Append a timestamp to this handler's log file. + This property may also accept a date format as described in the `datefmt <#.logging[].handlers[].datefmt>`__ property. + If the handler's `name <#.logging[].handlers[].name>`__ property is set to ``filename.log`` and this property is set to ``true`` or to a specific timestamp format, the resulting log file will be ``filename_.log``. + + +--------------------------- +The ``filelog`` log handler +--------------------------- + +This handler is meant primarily for performance logging and logs the performance of a regression test in one or more files. +The additional properties for the ``filelog`` handler are the following: + + +.. js:attribute:: .logging[].handlers[].basedir + +.. js:attribute:: .logging[].handlers_perflog[].basedir + +.. envvar:: RFM_PERFLOG_DIR + +.. option:: --perflogdir + + :required: No + :default: ``"./perflogs"`` + + The base directory of performance data log files. + + +.. js:attribute:: .logging[].handlers[].prefix + +.. js:attribute:: .logging[].handlers_perflog[].prefix + + :required: Yes + + This is a directory prefix (usually dynamic), appended to the `basedir <#.logging[].handlers_perflog[].basedir>`__, where the performance logs of a test will be stored. + This attribute accepts any of the check-specific `formatting placeholders <#.logging[].handlers_perflog[].format>`__. + This allows to create dynamic paths based on the current system, partition and/or programming environment a test executes with. + For example, a value of ``%(check_system)s/%(check_partition)s`` would generate the following structure of performance log files: + + + .. code-block:: none + + {basedir}/ + system1/ + partition1/ + test_name.log + partition2/ + test_name.log + ... + system2/ + ... + + +.. js:attribute:: .logging[].handlers[].append + +.. js:attribute:: .logging[].handlers_perflog[].append + + :required: No + :default: ``true`` + + Open each log file in append mode. + + +--------------------------- +The ``graylog`` log handler +--------------------------- + +This handler sends log records to a `Graylog `__ server. +The additional properties for the ``graylog`` handler are the following: + +.. js:attribute:: .logging[].handlers[].address + +.. js:attribute:: .logging[].handlers_perflog[].address + +.. envvar:: RFM_GRAYLOG_SERVER + + :required: Yes + + The address of the Graylog server defined as ``host:port``. + + +.. js:attribute:: .logging[].handlers[].extras + +.. js:attribute:: .logging[].handlers_perflog[].extras + + :required: No + :default: ``{}`` + + A set of optional key/value pairs to be passed with each log record to the server. + These may depend on the server configuration. + + +This log handler uses internally `pygelf `__. +If ``pygelf`` is not available, this log handler will be ignored. +`GELF `__ is a format specification for log messages that are sent over the network. +The ``graylog`` handler sends log messages in JSON format using an HTTP POST request to the specified address. +More details on this log format may be found `here `__. +An example configuration of this handler for performance logging is shown here: + +.. code:: python + + { + 'type': 'graylog', + 'address': 'graylog-server:12345', + 'level': 'info', + 'format': '%(message)s', + 'extras': { + 'facility': 'reframe', + 'data-version': '1.0' + } + } + + +Although the ``format`` is defined for this handler, it is not only the log message that will be transmitted the Graylog server. +This handler transmits the whole log record, meaning that all the information will be available and indexable at the remote end. + + +-------------------------- +The ``stream`` log handler +-------------------------- + +This handler sends log records to a file stream. +The additional properties for the ``stream`` handler are the following: + + +.. js:attribute:: .logging[].handlers[].name + +.. js:attribute:: .logging[].handlers_perflog[].name + + :required: No + :default: ``"stdout"`` + + The name of the file stream to send records to. + There are only two available streams: + + - ``stdout``: the standard output. + - ``stderr``: the standard error. + + +-------------------------- +The ``syslog`` log handler +-------------------------- + +This handler sends log records to UNIX syslog. +The additional properties for the ``syslog`` handler are the following: + + +.. js:attribute:: .logging[].handlers[].socktype + +.. js:attribute:: .logging[].handlers_perflog[].socktype + + :required: No + :default: ``"udp"`` + + The socket type where this handler will send log records to. + There are two socket types: + + - ``udp``: A UDP datagram socket. + - ``tcp``: A TCP stream socket. + + +.. js:attribute:: .logging[].handlers[].facility + +.. js:attribute:: .logging[].handlers_perflog[].facility + + :required: No + :default: ``"user"`` + + The Syslog facility where this handler will send log records to. + The list of supported facilities can be found `here `__. + + +.. js:attribute:: .logging[].handlers[].address + +.. js:attribute:: .logging[].handlers_perflog[].address + + :required: Yes + + The socket address where this handler will connect to. + This can either be of the form ``:`` or simply a path that refers to a Unix domain socket. + + + +Scheduler Configuration +----------------------- + +A scheduler configuration object contains configuration options specific to the scheduler's behavior. + + +------------------------ +Common scheduler options +------------------------ + + +.. js:attribute:: .schedulers[].name + + :required: Yes + + The name of the scheduler that these options refer to. + It can be any of the supported job scheduler `backends <#.systems[].partitions[].scheduler>`__. + + +.. js:attribute:: .schedulers[].job_submit_timeout + + :required: No + :default: 60 + + Timeout in seconds for the job submission command. + If timeout is reached, the regression test issuing that command will be marked as a failure. + + +.. js:attribute:: .schedulers[].target_systems + + :required: No + :default: ``["*"]`` + + A list of systems or system/partitions combinations that this scheduler configuration is valid for. + For a detailed description of this property, you may refer `here <#.environments[].target_systems>`__. + + + +Execution Mode Configuration +---------------------------- + +ReFrame allows you to define groups of command line options that are collectively called *execution modes*. +An execution mode can then be selected from the command line with the ``-mode`` option. +The options of an execution mode will be passed to ReFrame as if they were specified in the command line. + + +.. js:attribute:: .modes[].name + + :required: Yes + + The name of this execution mode. + This can be used with the ``-mode`` command line option to invoke this mode. + + +.. js:attribute:: .modes[].options + + :required: No + :default: ``[]`` + + The command-line options associated with this execution mode. + + +.. js:attribute:: .schedulers[].target_systems + + :required: No + :default: ``["*"]`` + + A list of systems or system/partitions combinations that this execution mode is valid for. + For a detailed description of this property, you may refer `here <#.environments[].target_systems>`__. + + + +General Configuration +--------------------- + +.. js:attribute:: .general[].check_search_path + +.. envvar:: RFM_CHECK_SEARCH_PATH + +.. option:: -c PATH | --checkpath PATH + + :required: No + :default: ``["${RFM_INSTALL_PREFIX}/checks/"]`` + + A list of paths (files or directories) where ReFrame will look for regression test files. + If the search path is set through the environment variable, it should be a colon separated list. + If specified from command line, the search path is constructed by specifying multiple times the command line option. + + +.. js:attribute:: .general[].check_search_recursive + +.. envvar:: RFM_CHECK_SEARCH_RECURSIVE + +.. option:: -R | --recursive + + :required: No + :default: ``true`` + + Search directories in the `search path <#.general[].check_search_path>`__ recursively. + + + +.. js:attribute:: .general[].colorize + +.. envvar:: RFM_COLORIZE + +.. option:: --nocolor + + :required: No + :default: ``true`` + + Use colors in output. + The command-line option sets the configuration option to ``false``. + + +.. js:attribute:: .general[].ignore_check_conflicts + +.. envvar:: RFM_IGNORE_CHECK_CONFLICTS + +.. option:: --ignore-check-conflicts + + :required: No + :default: ``false`` + + Ignore test name conflicts when loading tests. + + +.. js:attribute:: .general[].keep_stage_files + +.. envvar:: RFM_KEEP_STAGE_FILES + +.. option:: --keep-stage-files + + :required: No + :default: ``false`` + + Keep stage files of tests even if they succeed. + + +.. js:attribute:: .general[].module_map_file + +.. envvar:: RFM_MODULE_MAP_FILE + +.. option:: --module-mappings + + :required: No + :default: ``""`` + + File containing `module mappings `__. + + +.. js:attribute:: .general[].module_mappings + +.. envvar:: RFM_MODULE_MAPPINGS + +.. option:: -M MAPPING | --map-module MAPPING + + :required: No + :default: ``[]`` + + A list of `module mappings `__. + If specified through the environment variable, the mappings must be separated by commas. + If specified from command line, multiple module mappings are defined by passing the command line option multiple times. + + +.. js:attribute:: .general[].non_default_craype + +.. envvar:: RFM_NON_DEFAULT_CRAYPE + +.. option:: --non-default-craype + + :required: No + :default: ``false`` + + Test a non-default Cray Programming Environment. + This will emit some special instructions in the generated build and job scripts. + For more details, you may refer `here `__. + + +.. js:attribute:: .general[].purge_environment + +.. envvar:: RFM_PURGE_ENVIRONMENT + +.. option:: --purge-env + + :required: No + :default: ``false`` + + Purge any loaded environment modules before running any tests. + + +.. js:attribute:: .general[].save_log_files + +.. envvar:: RFM_SAVE_LOG_FILES + +.. option:: --save-log-files + + :required: No + :default: ``false`` + + Save any log files generated by ReFrame to its output directory + + +.. js:attribute:: .general[].target_systems + + :required: No + :default: ``["*"]`` + + A list of systems or system/partitions combinations that these general options are valid for. + For a detailed description of this property, you may refer `here <#.environments[].target_systems>`__. + + +.. js:attribute:: .general[].timestamp_dirs + +.. envvar:: RFM_TIMESTAMP_DIRS + +.. option:: --timestamp [TIMEFMT] + + :required: No + :default: ``""`` + + Append a timestamp to ReFrame directory prefixes. + Valid formats are those accepted by the `time.strftime() `__ function. + If specified from the command line without any argument, ``"%FT%T"`` will be used as a time format. + + +.. js:attribute:: .general[].unload_modules + +.. envvar:: RFM_UNLOAD_MODULES + +.. option:: -u MOD | --unload-module MOD + + :required: No + :default: ``[]`` + + A list of environment modules to unload before executing any test. + If specified using an the environment variable, a space separated list of modules is expected. + If specified from the command line, multiple modules can be passed by passing the command line option multiple times. + + +.. js:attribute:: .general[].user_modules + +.. envvar:: RFM_USER_MODULES + +.. option:: -m MOD | --module MOD + + :required: No + :default: ``[]`` + + A list of environment modules to be loaded before executing any test. + If specified using an the environment variable, a space separated list of modules is expected. + If specified from the command line, multiple modules can be passed by passing the command line option multiple times. + + +.. js:attribute:: .general[].verbose + +.. envvar:: RFM_VERBOSE + +.. option:: -v | --verbose + + :required: No + :default: 0 + + Increase the verbosity level of the output. + The higher the number, the more verbose the output will be. + If specified from the command line, the command line option must be specified multiple times to increase the verbosity level more than once. + + +Additional Environment Variables +-------------------------------- + +Here is a list of environment variables that do not have a configuration option counterpart. + + +.. envvar:: RFM_CONFIG_FILE + +.. option:: -C FILE | --config-file FILE + + The path to ReFrame's configuration file. + + +.. envvar:: RFM_SYSTEM + +.. option:: --system NAME + + The name of the system, whose configuration will be loaded. diff --git a/docs/configure.rst b/docs/configure.rst index b9f8dc3f07..055753c65d 100644 --- a/docs/configure.rst +++ b/docs/configure.rst @@ -2,437 +2,421 @@ Configuring ReFrame for Your Site ================================= -ReFrame provides an easy and flexible way to configure new systems and new programming environments. -By default, it ships with a generic local system configured. -This should be enough to let you run ReFrame on a local computer as soon as the basic `software requirements `__ are met. +ReFrame comes pre-configured with a minimal generic configuration that will allow you to run ReFrame on any system. +This will allow you to run simple local tests using the default compiler of the system. +Of course, ReFrame is much more powerful than that. +This section will guide you through configuring ReFrame for your HPC cluster. +We will use as a starting point a simplified configuration for the `Piz Daint `__ supercomputer at CSCS and we will elaborate along the way. -As soon as a new system with its programming environments is configured, adapting an existing regression test could be as easy as just adding the system's name in the :attr:`valid_systems ` list and its associated programming environments in the :attr:`valid_prog_environs ` list. +If you started using ReFrame from version 3.0, you can keep on reading this section, otherwise you are advised to have a look first at the `Migrating to ReFrame 3 `__ page. -The Configuration File ----------------------- -The configuration of systems and programming environments is performed by a special Python dictionary called ``site_configuration`` defined inside the file ``/reframe/settings.py``. +ReFrame's configuration file can be either a JSON file or Python file storing the site configuration in a JSON-formatted string. +The latter format is useful in cases that you want to generate configuration parameters on-the-fly, since ReFrame will import that Python file and the load the resulting configuration. +In the following we will use a Python-based configuration file also for historical reasons, since it was the only way to configure ReFrame in versions prior to 3.0. -The ``site_configuration`` dictionary should define two entries, ``systems`` and ``environments``. -The former defines the systems that ReFrame may recognize, whereas the latter defines the available programming environments. -The following example shows a minimal configuration for the `Piz Daint `__ supercomputer at CSCS: +Locating the Configuration File +------------------------------- -.. code-block:: python - - site_configuration = { - 'systems': { - 'daint': { - 'descr': 'Piz Daint', - 'hostnames': ['daint'], - 'modules_system': 'tmod', - 'partitions': { - 'login': { - 'scheduler': 'local', - 'modules': [], - 'access': [], - 'environs': ['PrgEnv-cray', 'PrgEnv-gnu', - 'PrgEnv-intel', 'PrgEnv-pgi'], - 'descr': 'Login nodes', - 'max_jobs': 4 - }, - - 'gpu': { - 'scheduler': 'nativeslurm', - 'modules': ['daint-gpu'], - 'access': ['--constraint=gpu'], - 'environs': ['PrgEnv-cray', 'PrgEnv-gnu', - 'PrgEnv-intel', 'PrgEnv-pgi'], - 'container_platforms': { - 'Singularity': { - 'modules': ['Singularity'] - } - }, - 'descr': 'Hybrid nodes (Haswell/P100)', - 'max_jobs': 100 - }, - - 'mc': { - 'scheduler': 'nativeslurm', - 'modules': ['daint-mc'], - 'access': ['--constraint=mc'], - 'environs': ['PrgEnv-cray', 'PrgEnv-gnu', - 'PrgEnv-intel', 'PrgEnv-pgi'], - 'container_platforms': { - 'Singularity': { - 'modules': ['Singularity'] - } - }, - 'descr': 'Multicore nodes (Broadwell)', - 'max_jobs': 100 - } - } - } - }, - - 'environments': { - '*': { - 'PrgEnv-cray': { - 'modules': ['PrgEnv-cray'], - }, - - 'PrgEnv-gnu': { - 'modules': ['PrgEnv-gnu'], - }, - - 'PrgEnv-intel': { - 'modules': ['PrgEnv-intel'], - }, - - 'PrgEnv-pgi': { - 'modules': ['PrgEnv-pgi'], - } - } - } - } - -System Configuration --------------------- - -The list of supported systems is defined as a set of key/value pairs under key ``systems``. -Each system is a key/value pair, with the key being the name of the system and the value being another set of key/value pairs defining its attributes. -The valid attributes of a system are the following: - -* ``descr``: A detailed description of the system (default is the system name). -* ``hostnames``: This is a list of hostname patterns according to the `Python Regular Expression Syntax `__ , which will be used by ReFrame when it tries to `auto-detect <#system-auto-detection>`__ the current system (default ``[]``). -* ``modules_system``: *[new in 2.8]* The modules system that should be used for loading environment modules on this system (default :class:`None`). - Three types of modules systems are currently supported: - - - ``tmod`` or ``tmod32``: The classic Tcl implementation of the `environment modules `__ (version 3.2). - - ``tmod31``: *[new in 2.21]* The classic Tcl implementation of the `environment modules `__ (version 3.1). - - ``tmod4``: The version 4 of the Tcl implementation of the `environment modules `__ (versions older than 4.1 are not supported). - - ``lmod``: The Lua implementation of the `environment modules `__. - -* ``modules``: *[new in 2.19]* Modules to be loaded always when running on this system. - These modules modify the ReFrame environment. - This is useful when for example a particular module is needed to submit jobs on a specific system. -* ``variables``: *[new in 2.19]* Environment variables to be set always when running on this system. -* ``prefix``: Default regression prefix for this system (default ``.``). -* ``stagedir``: Default stage directory for this system (default :class:`None`). -* ``outputdir``: Default output directory for this system (default :class:`None`). -* ``perflogdir``: Default directory prefix for storing performance logs for this system (default :class:`None`). -* ``resourcesdir``: Default directory for storing large resources (e.g., input data files, etc.) needed by regression tests for this system (default ``.``). -* ``partitions``: A set of key/value pairs defining the partitions of this system and their properties (default ``{}``). - Partition configuration is discussed in the `next section <#partition-configuration>`__. - -For a more detailed description of the ``prefix``, ``stagedir``, ``outputdir`` and ``perflogdir`` directories, please refer to the `"Configuring ReFrame Directories" `__ and `"Performance Logging" `__ sections. - -.. note:: - A different backend is used for Tmod 3.1, due to its different Python bindings. - -.. warning:: - .. versionchanged:: 2.18 - The ``logdir`` key is no more supported; please use ``perflogdir`` instead. - -Partition Configuration ------------------------ - -From the ReFrame's point of view, each system consists of a set of logical partitions. -These partitions need not necessarily correspond to real scheduler partitions. -For example, Piz Daint on the above example is split in *virtual partitions* using Slurm constraints. -Other systems may be indeed split into real scheduler partitions. - -The partitions of a system are defined similarly to systems as a set of key/value pairs with the key being the partition name and the value being another set of key/value pairs defining the partition's attributes. -The available partition attributes are the following: - -* ``descr``: A detailed description of the partition (default is the partition name). - -* ``scheduler``: The job scheduler and parallel program launcher combination that is used on this partition to launch jobs. - The syntax of this attribute is ``+``. - A list of the supported `schedulers <#supported-scheduler-backends>`__ and `parallel launchers <#supported-parallel-launchers>`__ can be found at the end of this section. - -* ``access``: A list of scheduler options that will be passed to the generated job script for gaining access to that logical partition (default ``[]``). - -* ``environs``: A list of environments, with which ReFrame will try to run any regression tests written for this partition (default ``[]``). - The environment names must be resolved inside the ``environments`` section of the ``site_configuration`` dictionary (see `Environments Configuration <#environments-configuration>`__ for more information). +ReFrame looks for a configuration file in the following locations in that order: -* ``container_platforms``: *[new in 2.20]* A set of key/value pairs specifying the supported container platforms for this partition and how their environment is set up. - Supported platform names are the following (names are case sensitive): +1. ``${HOME}/.reframe/settings.{py,json}`` +2. ``${RFM_INSTALL_PREFIX}/settings.{py,json}`` +3. ``/etc/reframe.d/settings.{py,json}`` - - ``Docker``: The `Docker `__ container runtime. - - ``Singularity``: The `Singularity `__ container runtime. - - ``Sarus``: The `Sarus `__ container runtime. +If both ``settings.py`` and ``settings.json`` are found, the Python file is preferred. +The ``RFM_INSTALL_PREFIX`` variable refers to the installation directory of ReFrame or the top-level source directory if you are running ReFrame from source. +Users have no control over this variable. +It is always set by the framework upon startup. - Each configured container runtime is associated optionally with an environment (modules and environment variables) that is providing it. - This environment is specified as a dictionary in the following format: +If no configuration file is found in any of the predefined locations, ReFrame will fall back to a generic configuration that allows it to run on any system. +You can find this generic configuration file `here `__. +Users may *not* modify this file. - .. code:: python +There are two ways to provide a custom configuration file to ReFrame: - { - 'modules': ['mod1', 'mod2', ...] - 'variables': {'ENV1': 'VAL1', 'ENV2': 'VAL2', ...} - } +1. Pass it through the ``-C`` or ``--config-file`` option. +2. Specify it using the ``RFM_CONFIG_FILE`` environment variable. +Command line options take always precedence over their respective environment variables. - If no special environment arrangement is needed for a configured container platform, you can simply specify an empty dictionary as an environment configuration, as it is shown in the following example: - .. code:: python +Anatomy of the Configuration File +--------------------------------- - 'container_platforms': { - 'Docker': {} - } +The whole configuration of ReFrame is a single JSON object whose properties are responsible for configuring the basic aspects of the framework. +We'll refer to these top-level properties as *sections*. +These sections contain other objects which further define in detail the framework's behavior. +If you are using a Python file to configure ReFrame, this big JSON configuration object is stored in a special variable called ``site_configuration``. +We will explore the basic configuration of ReFrame through the following configuration file that permits ReFrame to run on Piz Daint. +For the complete listing and description of all configuration options, you should refer to the `Configuration Reference `__. -* ``modules``: A list of modules to be loaded before running a regression test on that partition (default ``[]``). +.. literalinclude:: ../tutorial/config/settings.py + :lines: 10- -* ``variables``: A set of environment variables to be set before running a regression test on that partition (default ``{}``). - Environment variables can be set as follows (notice that both the variable name and its value are strings): +There are three required sections that each configuration file must provide: ``systems``, ``environments`` and ``logging``. +We will first cover these and then move on to the optional ones. - .. code-block:: python - 'variables': { - 'MYVAR': '3', - 'OTHER': 'foo' - } +--------------------- +Systems Configuration +--------------------- -* ``max_jobs``: The maximum number of concurrent regression tests that may be active (not completed) on this partition. +ReFrame allows you to configure multiple systems in the same configuration file. +Each system is a different object inside the ``systems`` section. +In our example we define only one system, namely Piz Daint: + +.. literalinclude:: ../tutorial/config/settings.py + :lines: 11-75 + +Each system is associated with a set of properties, which in this case are the following (for a complete list of properties, refer to the `configuration reference `__): + +* ``name``: The name of the system. + This should be an alphanumeric string (dashes ``-`` are allowed) and it will be used to refer to this system in other contexts. +* ``descr``: A detailed description of the system. +* ``hostnames``: This is a list of hostname patterns following the `Python Regular Expression Syntax `__, which will be used by ReFrame when it tries to automatically select a configuration entry for the current system. +* ``modules_system``: The environment modules system that should be used for loading environment modules on this system. + In this case, the classic Tcl implementation of the `environment modules `__. +* ``partitions``: The list of partitions that are defined for this system. + Each partition is defined as a separate object. + We devote the rest of this section in system partitions, since they are an essential part of ReFrame's configuration. + +A system partition in ReFrame is not bound to a real scheduler partition. +It is a virtual partition or separation of the system. +In the example shown here, we define three partitions that none of them corresponds to a scheduler partition. +The ``login`` partition refers to the login nodes of the system, whereas the ``gpu`` and ``mc`` partitions refer to two different set of nodes in the same cluster that are effectively separated using Slurm constraints. +Let's pick the ``gpu`` partition and look into it in more detail: + +.. literalinclude:: ../tutorial/config/settings.py + :lines: 31-51 + +The basic properties of a partition are the following: + +* ``name``: The name of the partition. + This should be an alphanumeric string (dashes ``-`` are allowed) and it will be used to refer to this partition in other contexts. +* ``descr``: A detailed description of the system partition. +* ``scheduler``: The workload manager (job scheduler) used in this partition for launching parallel jobs. + In this particular example, the `Slurm `__ scheduler is used. + For a complete list of the supported job schedulers, see `here `__. +* ``launcher``: The parallel job launcher used in this partition. + In this case, the ``srun`` command will be used. + For a complete list of the supported parallel job launchers, see `here `__. +* ``access``: A list of scheduler options that will be passed to the generated job script for gaining access to that logical partition. + Notice how in this case, the nodes are selected through a constraint and not an actual scheduler partition. +* ``environs``: The list of environments that ReFrame will use to run regression tests on this partition. + These are just symbolic names that refer to environments defined in the ``environments`` section described below. +* ``container_platforms``: A set of supported container platforms in this partition. + Each container platform is an object with a name and list of environment modules to load, in order to enable this platform. + For a complete list of the supported container platforms, see `here `__. +* ``max_jobs``: The maximum number of concurrent regression tests that may be active (i.e., not completed) on this partition. This option is relevant only when ReFrame executes with the `asynchronous execution policy `__. -* ``resources``: A set of custom resource specifications and how these can be requested from the partition's scheduler (default ``{}``). - - This variable is a set of key/value pairs with the key being the resource name and the value being a list of options to be passed to the partition's job scheduler. - The option strings can contain *placeholders* of the form ``{placeholder_name}``. - These placeholders may be replaced with concrete values by a regression tests through the :attr:`extra_resources` attribute. - - For example, one could define a ``gpu`` resource for a multi-GPU system that uses Slurm as follows: - - .. code-block:: python - 'resources': { - 'gpu': ['--gres=gpu:{num_gpus_per_node}'] - } - - A regression test then may request this resource as follows: - - .. code-block:: python - - self.extra_resources = {'gpu': {'num_gpus_per_node': '8'}} - - And the generated job script will have the following line in its preamble: - - .. code-block:: bash +-------------------------- +Environments Configuration +-------------------------- - #SBATCH --gres=gpu:8 +We have seen already environments to be referred to by the ``environs`` property of a partition. +An environment in ReFrame is simply a collection of environment modules, environment variables and compiler and compiler flags definitions. +None of these attributes is required. +An environment can simply by empty, in which case it refers to the actual environment that ReFrame runs in. +In fact, this is what the generic fallback configuration of ReFrame does. - A resource specification may also start with ``#PREFIX``, in which case ``#PREFIX`` will replace the standard job script prefix of the backend scheduler of this partition. - This is useful in cases of job schedulers like Slurm, that allow alternative prefixes for certain features. - An example is the `DataWarp `__ functionality of Slurm which is supported by the ``#DW`` prefix. - One could then define DataWarp related resources as follows: +Environments in ReFrame are configured under the ``environments`` section of the documentation. +In our configuration example for Piz Daint, we define each ReFrame environment to correspond to each of the Cray-provided programming environments. +In other systems, you could define a ReFrame environment to wrap a toolchain (MPI + compiler combination): - .. code-block:: python +.. literalinclude:: ../tutorial/config/settings.py + :lines: 76-93 - 'resources': { - 'datawarp': [ - '#DW jobdw capacity={capacity} access_mode={mode} type=scratch', - '#DW stage_out source={out_src} destination={out_dst} type={stage_filetype}' - ] - } +Each environment is associated with a name. +This name will be used to reference this environment in different contexts, as for example in the ``environs`` property of the system partitions. +This environment definition is minimal, since the default values for the rest of the properties serve our purpose. +For a complete list of the environment properties, see the `configuration reference `__. - A regression test that wants to make use of that resource, it can set its :attr:`extra_resources` as follows: +An important feature in ReFrame's configuration, is that you can define section objects differently for different systems or system partitions. +In the following, for demonstration purposes, we define ``PrgEnv-gnu`` differently for the ``mc`` partition of the ``daint`` system (notice the condensed form of writing this as ``daint:mc``): - .. code-block:: python +.. code-block:: python - self.extra_resources = { - 'datawarp': { - 'capacity': '100GB', - 'mode': 'striped', - 'out_src': '$DW_JOB_STRIPED/name', - 'out_dst': '/my/file', - 'stage_filetype': 'file' + { + 'name': 'PrgEnv-gnu', + 'modules': ['PrgEnv-gnu', 'openmpi'], + 'cc': 'mpicc', + 'cxx': 'mpicxx', + 'ftn': 'mpif90', + 'target_systems': ['daint:mc'] } - } - -.. note:: - For the `PBS <#supported-scheduler-backends>`__ and `Torque <#supported-scheduler-backends>`__ backends, options accepted in the ``access`` and ``resources`` attributes may either refer to actual ``qsub`` options or be just resources specifications to be passed to the ``-l`` option. - The backend assumes a ``qsub`` option, if the options passed in these attributes start with a ``-``. - -.. note:: - .. versionchanged:: 2.8 - A new syntax for the ``scheduler`` values was introduced as well as more parallel program launchers. - The old values for the ``scheduler`` key will continue to be supported. - -.. note:: - .. versionchanged:: 2.9 - Better support for custom job resources. - -.. note:: - .. versionchanged:: 2.14 - The ``modules`` and ``variables`` partition configuration parameters do not affect the ReFrame environment anymore. - They essentially define an environment to be always emitted when building and/or running the test on this partition. - If you want to modify the environment ReFrame runs in for a particular system, define these parameters inside the `system configuration <#system-configuration>`__. - - -Supported scheduler backends -============================ - -ReFrame supports the following job schedulers: - - -* ``slurm``: Jobs on the configured partition will be launched using `Slurm `__. - This scheduler relies on job accounting (``sacct`` command) in order to reliably query the job status. -* ``squeue``: *[new in 2.8.1]* - Jobs on the configured partition will be launched using `Slurm `__, but no job accounting is required. - The job status is obtained using the ``squeue`` command. - This scheduler is less reliable than the one based on the ``sacct`` command, but the framework does its best to query the job state as reliably as possible. - -* ``pbs``: *[new in 2.13]* Jobs on the configured partition will be launched using the `PBS Pro `__ scheduler. -* ``torque``: *[new in 3.0]* Jobs on the configured partition will be launched using the `Torque `__ scheduler. -* ``local``: Jobs on the configured partition will be launched locally as OS processes. +This environment loads different modules and sets the compilers differently, but the most important part is the ``target_systems`` property. +This property is a list of systems or system/partition combinations (as in this case) where this definition of the environment is in effect. +This means that ``PrgEnv-gnu`` will defined this way only for regression tests running on ``daint:mc``. +For all the other systems, it will be defined as shown before. -Supported parallel launchers -============================ -ReFrame supports the following parallel job launchers: +--------------------- +Logging configuration +--------------------- -* ``srun``: Programs on the configured partition will be launched using a bare ``srun`` command *without* any job allocation options passed to it. - This launcher may only be used with the ``slurm`` scheduler. -* ``srunalloc``: Programs on the configured partition will be launched using the ``srun`` command *with* job allocation options passed automatically to it. - This launcher may also be used with the ``local`` scheduler. -* ``alps``: Programs on the configured partition will be launched using the ``aprun`` command. -* ``mpirun``: Programs on the configured partition will be launched using the ``mpirun`` command. -* ``mpiexec``: Programs on the configured partition will be launched using the ``mpiexec`` command. -* ``ibrun``: *[new in 2.21]* Programs on the configured partition will be launched using the ``ibrun`` command. - This is a custom parallel job launcher used at `TACC `__. -* ``local``: Programs on the configured partition will be launched as-is without using any parallel program launcher. -* ``ssh``: *[new in 2.20]* Programs on the configured partition will be launched using SSH. - This option uses the partition's ``access`` parameter (see `above <#partition-configuration>`__) in order to determine the remote host and any additional options to be passed to the SSH client. - The ``ssh`` command will be launched in "batch mode," meaning that password-less access to the remote host must be configured. - Here is an example configuration for the ``ssh`` launcher: +ReFrame has a powerful logging mechanism that gives fine grained control over what information is being logged, where it is being logged and how this information is formatted. +Additionally, it allows for logging performance data from performance tests into different channels. +Let's see how logging is defined in our example configuration, which also represents a typical one for logging: - .. code:: python +.. literalinclude:: ../tutorial/config/settings.py + :lines: 94-130 - 'partition_name': { - 'scheduler': 'local+ssh', - 'access': ['-l admin', 'remote.host'], - 'environs': ['builtin'], - } +Logging is configured under the ``logging`` section of the configuration, which is a list of logger objects. +Unless you want to configure logging differently for different systems, a single logger object is enough. +Each logger object is associated with a logging level stored in the ``level`` property and has a set of logging handlers that are actually responsible for handling the actual logging records. +ReFrame's output is performed through the logging mechanism, meaning that if you don't specify any logging handler, you will not get any output from ReFrame! +The ``handlers`` property of the logger object holds the actual handlers. +Notice that you can use multiple handlers at the same time, which enables you to feed ReFrame's output to different sinks and at different verbosity levels. +All handler objects share a set of common properties. +These are the following: - Note that the environment is not propagated to the remote host, so the ``environs`` variable has no practical meaning except for enabling the testing of this partition. +* ``type``: This is the type of the handler, which determines its functionality. + Depending on the handler type, handler-specific properties may be allowed or required. +* ``level``: The cut-off level for messages reaching this handler. + Any message with a lower level number will be filtered out. +* ``format``: A format string for formatting the emitted log record. + ReFrame uses the format specifiers from `Python Logging `__, but also defines its owns specifiers. +* ``datefmt``: A time format string for formatting timestamps. + There are two log record fields that are considered timestamps: (a) ``asctime`` and (b) ``check_job_completion_time``. + ReFrame follows the time formatting syntax of Python's `time.strftime() `__ with a small tweak allowing full RFC3339 compliance when formatting time zone differences (see the `configuration reference `__ for more details). +We will not go into the details of the individual handlers here. +In this particular example we use three handlers of two distinct types: -There exist also the following aliases for specific combinations of job schedulers and parallel program launchers: +1. A file handler to print debug messages in the ``reframe.log`` file using a more extensive message format that contains a timestamp, the level name etc. +2. A stream handler to print any informational messages (and warnings and errors) from ReFrame to the standard output. + This handles essentially the actual output of ReFrame. +3. A file handler to print the framework's output in the ``reframe.out`` file. -* ``nativeslurm``: This is equivalent to ``slurm+srun``. -* ``local``: This is equivalent to ``local+local``. +It might initially seem confusing the fact that there are two ``level`` properties: one at the logger level and one at the handler level. +Logging in ReFrame works hierarchically. +When a message is logged, an log record is created, which contains metadata about the message being logged (log level, timestamp, ReFrame runtime information etc.). +This log record first goes into ReFrame's internal logger, where the record's level is checked against the logger's level (here ``debug``). +If the log record's level exceeds the log level threshold from the logger, it is forwarded to the logger's handlers. +Then each handler filters the log record differently and takes care of formatting the log record's message appropriately. +You can view logger's log level as a general cut off. +For example, if we have set it to ``warning``, no debug or informational messages would ever be printed. +Finally, there is a special set of handlers for handling performance log messages. +These are stored in the ``handlers_perflog`` property. +The performance handler in this example will create a file per test and per system/partition combination and will append the performance data to it every time the test is run. +Notice in the ``format`` property how the message to be logged is structured such that it can be easily parsed from post processing tools. +Apart from file logging, ReFrame offers more advanced performance logging capabilities through Syslog and Graylog. -Environments Configuration --------------------------- +For a complete description of the logging configuration properties and the different handlers, please refer to the `configuration reference `__. +Section `Running ReFrame `__ provides also examples of logging. -The environments available for testing in different systems are defined under the ``environments`` key of the top-level ``site_configuration`` dictionary. -The ``environments`` key is associated to a special dictionary that defines scopes for looking up an environment. The ``*`` denotes the global scope and all environments defined there can be used by any system. -Instead of ``*``, you can define scopes for specific systems or specific partitions by using the name of the system or partition. -For example, an entry ``daint`` will define a scope for a system called ``daint``, whereas an entry ``daint:gpu`` will define a scope for a virtual partition named ``gpu`` on the system ``daint``. -When an environment name is used in the ``environs`` list of a system partition (see `Partition Configuration <#partition-configuration>`__), it is first looked up in the entry of that partition, e.g., ``daint:gpu``. -If no such entry exists, it is looked up in the entry of the system, e.g., ``daint``. -If not found there, it is looked up in the global scope denoted by the ``*`` key. -If it cannot be found even there, an error will be issued. -This look up mechanism allows you to redefine an environment for a specific system or partition. -In the following example, we redefine ``PrgEnv-gnu`` for a system named ``foo``, so that whenever ``PrgEnv-gnu`` is used on that system, the module ``openmpi`` will also be loaded and the compiler variables should point to the MPI wrappers. -.. code-block:: python - - 'foo': { - 'PrgEnv-gnu': { - 'modules': ['PrgEnv-gnu', 'openmpi'], - 'cc': 'mpicc', - 'cxx': 'mpicxx', - 'ftn': 'mpif90', - } - } +----------------------------- +General configuration options +----------------------------- -An environment is also defined as a set of key/value pairs with the key being its name and the value being a dictionary of its attributes. -The possible attributes of an environment are the following: +General configuration options of the framework go under the ``general`` section of the configuration file. +This section is optional. +In this case, we define the search path for ReFrame test files to be the ``tutorial/`` subdirectory and we also instruct ReFrame to recursively search for tests there. +There are several more options that can go into this section, but the reader is referred to the `configuration reference `__ for the complete list. -* ``modules``: A list of modules to be loaded when this environment is used (default ``[]``, valid for all environment types) -* ``variables``: A set of variables to be set when this environment is used (default ``{}``, valid for all environment types) -* ``cc``: The C compiler (default: ``'cc'``) -* ``cxx``: The C++ compiler (default: ``'CC'``) -* ``ftn``: The Fortran compiler (default: ``'ftn'``) -* ``cppflags``: The default preprocessor flags (default: :class:`None`) -* ``cflags``: The default C compiler flags (default: :class:`None`) -* ``cxxflags``: The default C++ compiler flags (default: :class:`None`) -* ``fflags``: The default Fortran compiler flags (default: :class:`None`) -* ``ldflags``: The default linker flags (default: :class:`None`) -.. note:: - All flags for programming environments are now defined as list of strings instead of simple strings. +--------------------------- +Other configuration options +--------------------------- - .. versionchanged:: 2.17 +There are finally two more optional configuration sections that are not discussed here: -.. note:: - The ``type`` key is no more required for the environment configuration. +1. The ``schedulers`` section holds configuration variables specific to the different scheduler backends and +2. the ``modes`` section defines different execution modes for the framework. + Execution modes are discussed in `Running ReFrame `__. - .. versionchanged:: 2.22 -System Auto-Detection ---------------------- - -When ReFrame is launched, it tries to detect the current system and select the correct site configuration entry. The auto-detection process is as follows: +Picking a System Configuration +------------------------------ +As discussed previously, ReFrame's configuration file can store the configurations for multiple systems. +When launched, ReFrame will pick the first matching configuration and load it. +This process is performed as follows: ReFrame first tries to obtain the hostname from ``/etc/xthostname``, which provides the unqualified *machine name* in Cray systems. -If this cannot be found the hostname will be obtained from the standard ``hostname`` command. -Having retrieved the hostname, ReFrame goes through all the systems in its configuration and tries to match the hostname against any of the patterns in the ``hostnames`` attribute of `system configuration <#system-configuration>`__. -The detection process stops at the first match found, and the system it belongs to is considered as the current system. -If the system cannot be auto-detected, ReFrame will issue a warning and fall back to a generic system configuration, which is equivalent to the following: - -.. code-block:: python - - site_configuration = { - 'systems': { - 'generic': { - 'descr': 'Generic fallback system configuration', - 'hostnames': ['localhost'], - 'partitions': { - 'login': { - 'scheduler': 'local', - 'environs': ['builtin-gcc'], - 'descr': 'Login nodes' - } - } - } - }, - 'environments': { - '*': { - 'builtin-gcc': { - 'cc': 'gcc', - 'cxx': 'g++', - 'ftn': 'gfortran', - } - } - } - } - - - - -You can override completely the auto-detection process by specifying a system or a system partition with the ``--system`` option (e.g., ``--system daint`` or ``--system daint:gpu``). - -.. note:: - Instead of issuing an error, ReFrame falls back to a generic system configuration in case system auto-detection fails. - - .. versionchanged:: 2.19 - - - - -Viewing the current system configuration ----------------------------------------- - -.. versionadded:: 2.16 - -It is possible to ask ReFrame to print the configuration of the current system or the configuration of any programming environment defined for the current system. -There are two command-line options for performing these operations: - -* ``--show-config``: This option shows the current system's configuration and exits. - It can be combined with the ``--system`` option in order to show the configuration of another system. -* ``--show-config-env ENV``: This option shows the configuration of the programming environment ``ENV`` and exits. - The environment ``ENV`` must be defined for any of the partitions of the current system. - This option can also be combined with ``--system`` in order to show the configuration of a programming environment defined for another system. +If this cannot be found, the hostname will be obtained from the standard ``hostname`` command. +Having retrieved the hostname, ReFrame goes through all the systems in its configuration and tries to match the hostname against any of the patterns defined in each system's ``hostnames`` property. +The detection process stops at the first match found, and that system's configuration is selected. + +As soon as a system configuration is selected, all configuration objects that have a ``target_systems`` property are resolved against the selected system, and any configuration object that is not applicable is dropped. +So, internally, ReFrame keeps an *instantiation* of the site configuration for the selected system only. +To better understand this, let's assume that we have the following ``environments`` defined: + +.. code:: python + + 'environments': [ + { + 'name': 'PrgEnv-cray', + 'modules': ['PrgEnv-cray'] + }, + { + 'name': 'PrgEnv-gnu', + 'modules': ['PrgEnv-gnu'] + }, + { + 'name': 'PrgEnv-gnu', + 'modules': ['PrgEnv-gnu', 'openmpi'], + 'cc': 'mpicc', + 'cxx': 'mpicxx', + 'ftn': 'mpif90', + 'target_systems': ['foo'] + } + ], + + +If the selected system is ``foo``, then ReFrame will use the second definition of ``PrgEnv-gnu`` which is specific to the ``foo`` system. + +You can override completely the system auto-selection process by specifying a system or system/partition combination with the ``--system`` option, e.g., ``--system=daint`` or ``--system=daint:gpu``. + + +Querying Configuration Options +------------------------------ + +ReFrame offers the powerful ``--show-config`` command-line option that allows you to query any configuration parameter of the framework and see how it is set for the selected system. +Using no arguments or passing ``all`` to this option, the whole configuration for the currently selected system will be printed in JSON format, which you can then pipe to a JSON command line editor, such as `jq `__, and either get a colored output or even generate a completely new ReFrame configuration! + +Passing specific configuration keys in this option, you can query specific parts of the configuration. +Let's see some concrete examples: + +* Query the current system's partitions: + + .. code:: + + ./bin/reframe -C tutorial/config/settings.py --system=daint --show-config=systems/0/partitions + + .. code:: javascript + + [ + { + "name": "login", + "descr": "Login nodes", + "scheduler": "local", + "launcher": "local", + "environs": [ + "PrgEnv-cray", + "PrgEnv-gnu", + "PrgEnv-intel", + "PrgEnv-pgi" + ], + "max_jobs": 4 + }, + { + "name": "gpu", + "descr": "Hybrid nodes (Haswell/P100)", + "scheduler": "slurm", + "launcher": "srun", + "modules": [ + "daint-gpu" + ], + "access": [ + "--constraint=gpu" + ], + "environs": [ + "PrgEnv-cray", + "PrgEnv-gnu", + "PrgEnv-intel", + "PrgEnv-pgi" + ], + "container_platforms": [ + { + "name": "Singularity", + "modules": [ + "Singularity" + ] + } + ], + "max_jobs": 100 + }, + { + "name": "mc", + "descr": "Multicore nodes (Broadwell)", + "scheduler": "slurm", + "launcher": "srun", + "modules": [ + "daint-mc" + ], + "access": [ + "--constraint=mc" + ], + "environs": [ + "PrgEnv-cray", + "PrgEnv-gnu", + "PrgEnv-intel", + "PrgEnv-pgi" + ], + "container_platforms": [ + { + "name": "Singularity", + "modules": [ + "Singularity" + ] + } + ], + "max_jobs": 100 + } + ] + + Check how the output changes if we explicitly set system to ``daint:login``: + + .. code:: + + ./bin/reframe -C tutorial/config/settings.py --system=daint:login --show-config=systems/0/partitions + + + .. code:: javascript + + [ + { + "name": "login", + "descr": "Login nodes", + "scheduler": "local", + "launcher": "local", + "environs": [ + "PrgEnv-cray", + "PrgEnv-gnu", + "PrgEnv-intel", + "PrgEnv-pgi" + ], + "max_jobs": 4 + } + ] + + ReFrame will internally represent system ``daint`` as having a single partition only. + Notice also how you can use indexes to objects elements inside a list. + +* Query an environment configuration: + + .. code:: + + ./bin/reframe -C tutorial/config/settings.py --system=daint --show-config=environments/@PrgEnv-gnu + + .. code:: javascript + + { + "name": "PrgEnv-gnu", + "modules": [ + "PrgEnv-gnu" + ] + } + + If an object has a ``name`` property you can address it by name using the ``@name`` syntax, instead of its index. + +* Query an environment's compiler: + + .. code:: + + ./bin/reframe -C tutorial/config/settings.py --system=daint --show-config=environments/@PrgEnv-gnu/cxx + + .. code:: javascript + + "CC" + + Notice that although the C++ compiler is not defined in the environment's definitions, ReFrame will print the default value, if you explicitly query its value. diff --git a/docs/index.rst b/docs/index.rst index d348901eaf..e0f7ed6081 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -62,3 +62,4 @@ Publications About ReFrame Reference Guide Sanity Functions Reference + Configuration Reference diff --git a/docs/migration_2_to_3.rst b/docs/migration_2_to_3.rst index c14488bb0a..e57444c157 100644 --- a/docs/migration_2_to_3.rst +++ b/docs/migration_2_to_3.rst @@ -2,14 +2,66 @@ Migrating to ReFrame 3 ====================== +ReFrame 3 brings substantial changes in its configuration. +The configuration component was completely revised and rewritten from scratch in order to allow much more flexibility in how the framework's configuration options are handled, as well as to ensure the maintainability of the framework in the future. -Updating your tests +At the same time, ReFrame 3 deprecates some common pre-2.20 test syntax in favor of the more modern and intuitive pipeline hooks. + +This guide details the necessary steps in order to easily migrate to ReFrame 3. + + +Updating Your Site Configuration +-------------------------------- + +As described in `Configuring ReFrame for Your Site `__, ReFrame's configuration file has changed substantially. +However, you don't need to manually update your configuration; ReFrame will do that automatically for you. +As soon as it detects an old-style configuration file, it will convert it to the new syntax save it in a temporary file: + + +.. code-block:: none + + $ ./bin/reframe -C unittests/resources/settings_old_syntax.py -l + ./bin/reframe: the syntax of the configuration file 'unittests/resources/settings_old_syntax.py' is deprecated + ./bin/reframe: configuration file has been converted to the new syntax here: '/var/folders/h7/k7cgrdl13r996m4dmsvjq7v80000gp/T/tmph5n8u3kf.py' + +Alternatively, you can convert any old configuration file using the conversion tool ``tools/convert_config.py``: + +.. code-block:: none + + $ python3 tools/convert_config.py unittests/resources/settings_old_syntax.py + Conversion successful! The converted file can be found at '/var/folders/h7/k7cgrdl13r996m4dmsvjq7v80000gp/T/tmpz4f6yer4.py'. + + +Another important change is that default locations for looking up a configuration file has changed (see `Configuring ReFrame for Your Site `__ for more details). +That practically means that if you were relying on ReFrame loading your ``reframe/settings.py`` by default, this is no longer true. +You have to move it to any of the default settings locations or set the corresponding command line option or environment variable. + + +Automatic conversion limitations +================================ + +ReFrame does a pretty good job in converting correctly your old configuration files, but there are some limitations: + +- Your code formatting will be lost. + ReFrame will use its own, which is PEP8 compliant nonetheless. +- Any comments will be lost. +- Any code that was used to dynamically generate configuration parameters will be lost. + ReFrame will generate the new configuration based on what was the actual old configuration after any dynamic generation. + + + +.. note:: + + The very old logging configuration syntax (prior to ReFrame 2.13) is no more recognized and the configuration conversion tool does not take it into account. + + +Updating Your Tests ------------------- ReFrame 2.20 introduced a new powerful mechanism for attaching arbitrary functions hooks at the different pipeline stages. This mechanism provides an easy way to configure and extend the functionality of a test, eliminating essentially the need to override pipeline stages for this purpose. -ReFrame 3.0 deprecates the old practice for overriding pipeline stage methods in favor of using pipeline hooks. +ReFrame 3.0 deprecates the old practice of overriding pipeline stage methods in favor of using pipeline hooks. In the old syntax, it was quite common to override the ``setup()`` method, in order to configure your test based on the current programming environment or the current system partition. The following is a typical example of that: @@ -74,5 +126,44 @@ If you try to override the ``setup()`` method in any of the subclasses of ``MyEx Suppressing deprecation warnings ================================ -You can suppress any deprecation warning issued by ReFrame by passing the ``--no-deprecation-warnings`` flag. +Although not recommended, you can suppress any deprecation warning issued by ReFrame by passing the ``--no-deprecation-warnings`` flag. + + +Getting schedulers and launchers by name +======================================== + + +The way to get a scheduler or launcher instance by name has changed. +Prior to ReFrame 3, this was written as follows: + +.. code:: python + + from reframe.core.launchers.registry import getlauncher + + + class MyTest(rfm.RegressionTest): + ... + + @rfm.run_before('run') + def setlauncher(self): + self.job.launcher = getlauncher('local')() + + + +Now you have to simply replace the import statement with the following: + + +.. code:: python + + from reframe.core.backends import getlauncher + + +Similarly for schedulers, the ``reframe.core.schedulers.registry`` module must be replaced with ``reframe.core.backends``. + + +Other Changes +------------- +ReFrame 3.0-dev0 introduced a `change `__ in the way that a search path for checks was constructed in the command-line using the ``-c`` option. +ReFrame 3.0 reverts the behavior of the ``-c`` to its original one (i.e., ReFrame 2.x behavior), in which multiple paths can be specified by passing multiple times the ``-c`` option. +Overriding completely the check search path can be achieved in ReFrame 3.0 through the :envvar:`RFM_CHECK_SEARCH_PATH` environment variable or the corresponding configuration option. diff --git a/docs/reference.rst b/docs/reference.rst index f34fd9c9f1..43b43ea583 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -43,7 +43,7 @@ Job schedulers and parallel launchers :show-inheritance: -.. automodule:: reframe.core.launchers.registry +.. automodule:: reframe.core.backends :members: :show-inheritance: diff --git a/docs/requirements.txt b/docs/requirements.txt index 60b5338068..a9bb99f4c2 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -18,7 +18,7 @@ requests>=2.18.4 setuptools>=31.0.1 six>=1.11.0 snowballstemmer>=1.2.1 -Sphinx +Sphinx>=1.7.5 sphinx-autobuild>=0.7.1 sphinx-bootstrap-theme>=0.5.3 sphinx-fakeinv>=1.0.0 diff --git a/docs/running.rst b/docs/running.rst index 0fe0e3e27c..b266667c26 100644 --- a/docs/running.rst +++ b/docs/running.rst @@ -6,7 +6,7 @@ Before getting into any details, the simplest way to invoke ReFrame is the follo .. code-block:: bash - ./bin/reframe -c /path/to/checks -R --run + ./bin/reframe -c /path/to/checks -R -r This will search recursively for test files in ``/path/to/checks`` and will start running them on the current system. @@ -28,8 +28,9 @@ Currently there are only two available actions: 1. Listing of the selected checks 2. Execution of the selected checks +------------------------------- Listing of the regression tests -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +------------------------------- To retrieve a listing of the selected checks, you must specify the ``-l`` or ``--list`` options. This will provide a list with a brief description for each test containing only its name and the path to the file where it is defined. @@ -215,8 +216,9 @@ The detailed listing shows the description of the test, its supported systems an Previous versions were showing all the tests found. +--------------------------------- Execution of the regression tests -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +--------------------------------- To run the regression tests you should specify the *run* action though the ``-r`` or ``--run`` options. @@ -310,8 +312,9 @@ ReFrame the does not search recursively into directories specified with the ``-c The ``-c`` option completely overrides the default path. Currently, there is no option to prepend or append to the default regression path. -However, you can build your own check path by specifying a colon separated list of paths to the ``-c`` option. -The ``-c``\ option accepts also regular files. This is very useful when you are implementing new regression tests, since it allows you to run only your test: +However, you can build your own check path by specifying multiple times the ``-c`` option. +The ``-c`` option accepts also regular files. +This is very useful when you are implementing new regression tests, since it allows you to run only your test: .. code-block:: bash @@ -325,16 +328,6 @@ The ``-c``\ option accepts also regular files. This is very useful when you are .. versionadded:: 2.12 -.. warning:: - Using the command line ``-c`` or ``--checkpath`` multiple times is not supported anymore and only the last option will be considered. - Multiple paths should be passed instead as a colon separated list: - - .. code-block:: bash - - ./bin/reframe -c /path/to/my/first/test.py:/path/to/my/second/ -r - - - .. versionchanged:: 3.0 Filtering of Regression Tests @@ -343,8 +336,9 @@ Filtering of Regression Tests At this phase you can select which regression tests should be run or listed. There are several ways to select regression tests, which we describe in more detail here: +------------------------- Selecting tests by system -^^^^^^^^^^^^^^^^^^^^^^^^^ +------------------------- .. versionadded:: 2.15 @@ -373,8 +367,9 @@ Finally, in order to list all the tests found regardless of their supported syst ./bin/reframe --skip-system-check -l +------------------------------------------ Selecting tests by programming environment -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +------------------------------------------ To select tests by the programming environment, use the ``-p`` or ``--prgenv`` options: @@ -411,8 +406,9 @@ For example, the following will select all tests that support programming enviro The ``-p`` option recognizes regular expressions as arguments. +----------------------- Selecting tests by tags -^^^^^^^^^^^^^^^^^^^^^^^ +----------------------- As we have seen in the `"ReFrame tutorial" `__, every regression test may be associated with a set of tags. Using the ``-t`` or ``--tag`` option you can select the regression tests associated with a specific tag. @@ -430,8 +426,9 @@ The list of tags associated with a check can be viewed in the listing output whe The ``-t`` option recognizes regular expressions as arguments. +----------------------- Selecting tests by name -^^^^^^^^^^^^^^^^^^^^^^^ +----------------------- It is possible to select or exclude tests by name through the ``--name`` or ``-n`` and ``--exclude`` or ``-x`` options. For example, you can select only the ``Example7Test`` from the tutorial as follows: @@ -736,16 +733,14 @@ This is useful if you want to check the directories that ReFrame will create. You can also define different default directories per system by specifying them in the `site configuration `__ settings file. The command line options, though, take always precedence over any default directory. + Logging ------- -From version 2.4 onward, ReFrame supports logging of its actions. -ReFrame creates two files inside the current working directory every time it is run: - -* ``reframe.out``: This file stores the output of a run as it was printed in the standard output. -* ``reframe.log``: This file stores more detailed of information on ReFrame's actions. - -By default, the output in ``reframe.log`` looks like the following: +From version 2.4 onward, ReFrame not only supports logging of its actions, but its normal output is also fully configurable. +All output of ReFrame is handled by logging handlers, which can send information to multiple channels at once filtering it independently based on severity levels. +By default, using its generic configuration, ReFrame sends informational data to its standard output and debug data to ``reframe.log``. +The debug information in ``reframe.log`` looks like the following: .. code-block:: none @@ -787,180 +782,24 @@ By default, the output in ``reframe.log`` looks like the following: Each line starts with a timestamp, the level of the message (``info``, ``debug`` etc.), the context in which the framework is currently executing (either ``reframe`` or the name of the current test and, finally, the actual message. -Every time ReFrame is run, both ``reframe.out`` and ``reframe.log`` files will be rewritten. -However, you can ask ReFrame to copy them to the output directory before exiting by passing it the ``--save-log-files`` option. - -Configuring Logging -^^^^^^^^^^^^^^^^^^^ - -You can configure several aspects of logging in ReFrame and even how the output will look like. -ReFrame's logging mechanism is built upon Python's `logging `__ framework adding extra logging levels and more formatting capabilities. - -Logging in ReFrame is configured by the ``logging_config`` variable in the ``reframe/settings.py`` file. -The default configuration looks as follows: - -.. literalinclude:: ../reframe/settings.py - :lines: 47-74 - :dedent: 4 - -Note that this configuration dictionary is not the same as the one used by Python's logging framework. -It is a simplified version adapted to the needs of ReFrame. - -The ``logging_config`` dictionary has two main key entries: - -* ``level`` (default: ``'INFO'``): - This is the lowest level of messages that will be passed down to the different log record handlers. - Any message with a lower level than that, it will be filtered out immediately and will not be passed to any handler. - ReFrame defines the following logging levels with a decreasing severity: ``CRITICAL``, ``ERROR``, ``WARNING``, ``INFO``, ``VERBOSE`` and ``DEBUG``. - Note that the level name is *not* case sensitive in ReFrame. -* ``handlers``: - A list of log record handlers that are attached to ReFrame's logging mechanism. - You can attach as many handlers as you like. - For example, by default ReFrame uses three handlers: (a) a handler that logs debug information into ``reframe.log``, (b) a handler that controls the actual output of the framework to the standart output, which does not print any debug messages, and (c) a handler that writes the same output to a file ``reframe.out``. - -Each handler is configured by another dictionary that holds its properties as string key/value pairs. -For standard ReFrame logging there are currently two types of handlers, which recognize different properties. - -.. note:: - New syntax for handlers is introduced. - The old syntax is still valid, but users are advised to update their logging configuration to the new syntax. - - .. versionchanged:: 2.13 - - -Common Log Handler Attributes -""""""""""""""""""""""""""""" - -All handlers accept the following set of attributes (keys) in their configuration: - -* ``type``: (required) the type of the handler. - There are several types of handlers used for logging in ReFrame. - Some of them are only relevant for performance logging: - - 1. ``file``: a handler that writes log records in file. - 2. ``stream``: a handler that writes log records in a file stream. - 3. ``syslog``: a handler that sends log records to Unix syslog. - 4. ``filelog``: a handler for writing performance logs (relevant only for `performance logging <#performance-logging>`__). - 5. ``graylog``: a handler for sending performance logs to a Graylog server (relevant only for `performance logging <#performance-logging>`__). - - -* ``level``: (default: ``DEBUG``) The lowest level of log records that this handler can process. -* ``format`` (default: ``'%(message)s'``): Format string for the printout of the log record. - ReFrame supports all the `log record attributes `__ from Python's logging library and provides the following additional ones: - - - ``check_environ``: The programming environment a test is currently executing for. - - ``check_info``: Print live information of the currently executing check. - By default this field has the form `` on using ``. - It can be configured on a per test basis by overriding the :func:`info ` method of a specific regression test. - - ``check_jobid``: Prints the job or process id of the job or process associated with the currently executing regression test. - If a job or process is not yet created, ``-1`` will be printed. - - ``check_job_completion_time``: *[new in 2.21]* The completion time of the job spawned by this regression test. - This timestamp will be formatted according to ``datefmt`` (see below). - The accuracy of the timestamp depends on the backend scheduler. - The ``slurm`` scheduler backend relies on job accounting and returns the actual termination time of the job. - The rest of the backends report as completion time the moment when the framework realizes that the spawned job has finished. - In this case, the accuracy depends on the execution policy used. - If tests are executed with the serial execution policy, this is close to the real completion time, but if the asynchronous execution policy is used, it can differ significantly. - If the job completion time cannot be retrieved, ``None`` will be printed. - - ``check_job_completion_time_unix``: *[new in 3.0]* The completion time of the job spawned by this regression test expressed as UNIX time. - This is a raw time field and will not be formatted according to ``datefmt``. - If specific formatting is desired, the ``check_job_completion_time`` should be used instead. - - ``check_name``: Prints the name of the regression test on behalf of which ReFrame is currently executing. - If ReFrame is not in the context of regression test, ``reframe`` will be printed. - - ``check_num_tasks``: The number of tasks assigned to the regression test. - - ``check_outputdir``: The output directory associated with the currently executing test. - - ``check_partition``: The system partition where this test is currently executing. - - ``check_stagedir``: The stage directory associated with the currently executing test. - - ``check_system``: The host system where this test is currently executing. - - ``check_tags``: The tags associated with this test. - - ``osuser``: The name of the OS user running ReFrame. - - ``osgroup``: The group name of the OS user running ReFrame. - - ``version``: The ReFrame version. - -* ``datefmt`` (default: ``'%FT%T'``) The format that will be used for outputting timestamps (i.e., the ``%(asctime)s`` and the ``%(check_job_completion_time)s`` fields). - In addition to the format directives supported by the standard library's `time.strftime() `__ function, ReFrame allows you to use the ``%:z`` directive -- a GNU ``date`` extension -- that will print the time zone difference in a RFC3339 compliant way, i.e., ``+/-HH:MM`` instead of ``+/-HHMM``. - -.. caution:: - The ``testcase_name`` logging attribute is replaced with the ``check_info``, which is now also configurable - - .. versionchanged:: 2.10 - - -.. note:: - Support for fully RFC3339 compliant time zone formatting. - - .. versionadded:: 3.0 - - -File log handlers -""""""""""""""""" - -In addition to the common log handler attributes, file log handlers accept the following: - -* ``name``: (required) The name of the file where log records will be written. -* ``append`` (default: :class:`False`) Controls whether ReFrame should append to this file or not. -* ``timestamp`` (default: :class:`None`): Append a timestamp to this log filename. - This property may accept any date format that is accepted also by the ``datefmt`` property. - If the name of the file is ``filename.log`` and this attribute is set to ``True``, the resulting log file name will be ``filename_.log``. - - - -Stream log handlers -""""""""""""""""""" - -In addition to the common log handler attributes, file log handlers accept the following: - -* ``name``: (default ``stdout``) The symbolic name of the log stream to use. - Available values: ``stdout`` for standard output and ``stderr`` for standard error. +Every time ReFrame is run, the ``reframe.log`` files will be rewritten. +However, you can ask ReFrame to copy it to the output directory before exiting by passing it the ``--save-log-files`` option. - -Syslog log handler -"""""""""""""""""" - -In addition to the common log handler attributes, file log handlers accept the following: - -* ``socktype``: The type of socket where the handler will send log records to. There are two socket types: - - 1. ``udp``: (default) This opens a UDP datagram socket. - 2. ``tcp``: This opens a TCP stream socket. - -* ``facility``: (default: ``user``) The Syslog facility to send records to. - The list of supported facilities can be found `here `__. -* ``address``: (required) The address where the handler will connect to. - This can either be of the form ``:`` or simply a path that refers to a Unix domain socket. - - -.. note:: - .. versionadded:: 2.17 +Users have full control over what is being logged, where it is send and how log records are formatted. +The `configuration guide `__ gives an overview of the logging configuration, whereas a complete description of the available options can be found in the `configuration reference `__. +------------------- Performance Logging -^^^^^^^^^^^^^^^^^^^ +------------------- -ReFrame supports an additional logging facility for recording performance values, in order to be able to keep historical performance data. -This is configured by the ``perf_logging_config`` variables, whose syntax is the same as for the ``logging_config``: - -.. literalinclude:: ../reframe/settings.py - :lines: 76-95 - :dedent: 4 - -Performance logging introduces two new log record handlers, specifically designed for this purpose. - -File-based Performance Logging -"""""""""""""""""""""""""""""" - -The type of this handler is ``filelog`` and logs the performance of a regression test in one or more files. -The attributes of this handler are the following: - -* ``prefix``: This is the directory prefix (usually dynamic) where the performance logs of a test will be stored. - This attribute accepts any of the check-specific formatting placeholders described `above <#common-log-handler-attributes>`__. - This allows you to create dynamic paths based on the current system, partition and/or programming environment a test executes. - This dynamic prefix is appended to the "global" performance log directory prefix, configurable through the ``--perflogdir`` option or the ``perflogdir`` attribute of the `system configuration `__. - The default configuration of ReFrame for performance logging (shown in the previous listing) generates the following files: +Performance testing is an essential part of ReFrame and performance data from tests is handled specially. +ReFrame allows you to send performance data to multiple channels, such as files, log servers and system logs. +By default, performance data is sent to files organized per system, per partition and per test as follows: .. code-block:: none - {PERFLOG_PREFIX}/ + {perflog_basedir}/ system1/ partition1/ test_name.log @@ -970,41 +809,8 @@ The attributes of this handler are the following: system2/ ... - A log file, named after the test's name, is generated in different directories, which are themselves named after the system and partition names that this test has run on. - The ``PERFLOG_PREFIX`` will have the value of ``--perflogdir`` option, if specified, otherwise it will default to ``{REFRAME_PREFIX}/perflogs``. - You can always check its value by looking into the paths printed by ReFrame at the beginning of its output: - - .. code-block:: none - - Command line: ./reframe.py --prefix=/foo --system=generic -l - Reframe version: 2.13-dev0 - Launched by user: USER - Launched on host: HOSTNAME - Reframe paths - ============= - Check prefix : /Users/karakasv/Repositories/reframe - (R) Check search path : 'checks/' - Stage dir prefix : /foo/stage/ - Output dir prefix : /foo/output/ - Perf. logging prefix : /foo/perflogs - List of matched checks - ====================== - Found 0 check(s). - -* ``format``: The syntax of this attribute is the same as of the standard logging facility, except that it adds a couple more performance-specific formatting placeholders: - - - ``check_perf_lower_thres``: The lower threshold of the difference from the reference value expressed as a fraction of the reference. - - ``check_perf_upper_thres``: The upper threshold of the difference from the reference value expressed as a fraction of the reference. - - ``check_perf_ref``: The reference performance value of a certain performance variable. - - ``check_perf_value``: The performance value obtained by this test for a certain performance variable. - - ``check_perf_var``: The name of the `performance variable `__, whose value is logged. - - ``check_perf_unit``: The unit of measurement for the measured performance variable specified in the corresponding tuple of the :attr:`reframe.core.pipeline.RegressionTest.reference` attribute. - -.. note:: - .. versionchanged:: 2.20 - Support for logging `num_tasks` in performance logs was added. - -Using the default performance log format, the resulting log entries look like the following: +Whenever a test executes on a specific system/partition combination, the obtained performance data will be appended to a file. +A performance log record has the following format by default: .. code-block:: none @@ -1012,63 +818,47 @@ Using the default performance log format, the resulting log entries look like th 2019-10-23T13:46:27|reframe 2.20-dev2|Example7Test on daint:gpu using PrgEnv-gnu|jobid=813560|num_tasks=1|perf=50.737651|ref=50.0 (l=-0.1, u=0.1)|Gflop/s 2019-10-23T13:46:48|reframe 2.20-dev2|Example7Test on daint:gpu using PrgEnv-pgi|jobid=813561|num_tasks=1|perf=50.720164|ref=50.0 (l=-0.1, u=0.1)|Gflop/s -The interpretation of the performance values depends on the individual tests. -The above output is from the CUDA performance test we presented in the `tutorial `__, so the value refers to the achieved Gflop/s. +There is information about the actual test run and the performance data obtained for the different performance variables. +The log record format is fully configurable as well as the directory structure. +More information can be found in the `configuration reference `__. -Performance Logging Using Graylog -""""""""""""""""""""""""""""""""" +More Advanced Performance Logging +================================= -The type of this handler is ``graylog`` and it logs performance data to a `Graylog `__ server. -Graylog is a distributed enterprise log management service. -An example configuration of such a handler is the following: +ReFrame offers additional capabilities for performance logging. +You can instruct it to send data to a log management server, which provides a dedicated service for handling large amount of logs. +These services are usually combined with log analysis tools, such `Grafana `__ and `Kibana `__, which offer specialized functionality for log analysis. +If your site uses a centralized log management, you could feed the ReFrame performance logs, create dashboards and correlate it with other events on the system. -.. code-block:: python +There are two ways to achieve this in ReFrame, either through the `graylog `__ log handler or the `syslog `__ log handler. +The first one sends data to a `Graylog `__ server, whereas the second one sends data to a Syslog server. +A usual setup for centers is to feed all the syslog data to a centralized log management system, where the logs can be visualized and analyzed. - { - 'type': 'graylog', - 'host': 'my.graylog.server', - 'port': 12345, - 'level': 'INFO', - 'format': ( - '%(asctime)s|reframe %(version)s|' - '%(check_info)s|jobid=%(check_jobid)s|' - 'num_tasks=%(check_num_tasks)s|' - '%(check_perf_var)s=%(check_perf_value)s|' - 'ref=%(check_perf_ref)s ' - '(l=%(check_perf_lower_thres)s, ' - 'u=%(check_perf_upper_thres)s)' - ), - 'extras': { - 'facility': 'reframe', - } - }, +The following image is a snapshot of an actual dashboard visualizing `GROMACS `__ performance data from ReFrame over time. -This handler introduces three new attributes: -* ``host``: (required) The Graylog server that accepts the log messages. -* ``port``: (required) The port where the Graylog server accepts connections. -* ``extras``: (optional) A set of optional user attributes to be passed with each log record to the server. - These may depend on the server configuration. - -This log handler uses internally `pygelf `__, so this Python module must be available, otherwise this log handler will be ignored. -`GELF `__ is a format specification for log messages that are sent over the network. -The ReFrame's ``graylog`` handler sends log messages in JSON format using an HTTP POST request to the specified host and port. -More details on this log format may be found `here `__. +.. figure:: _static/img/gromacs-perf.png + :align: center + :alt: Visualization of GROMACS performance data from ReFrame over time. +----------------------------- Adjusting verbosity of output -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +----------------------------- -ReFrame's output is handled by a logging mechanism. -In fact, as revealed in the corresponding configuration entry (see `Configuring Logging <#configuring-logging>`__), a specific logging handler takes care of printing ReFrame's message in the standard output. -One way to change the verbosity level of the output is by explicitly setting the value of the ``level`` key in the configuration of the output handler. +As discussed in the `configuration guide `__, ReFrame's output is handled also by its logging mechanism. +Increasing or decreasing the verbosity of its output can be achieved by adjusting the level of the log handler that sends data to standard output. Alternatively, you may increase the verbosity level from the command line by chaining the ``-v`` or ``--verbose`` option. Every time ``-v`` is specified, the next verbosity level will be selected for the output. -For example, if the initial level of the output handler is set to ``INFO`` (in the configuration file), specifying ``-v`` twice will make ReFrame spit out all ``DEBUG`` messages. +For example, if the initial level of the output log handler is set to ``info`` (in the configuration file), specifying ``-v`` twice will make ReFrame spit out all ``debug`` messages. + +.. note:: + + .. versionadded:: 2.16 + + The ``--verbose`` option was added. -.. versionadded:: 2.16 - ``-v`` and ``--verbose`` options are added. Asynchronous Execution of Regression Checks @@ -1323,3 +1113,10 @@ Here is an example that shows how to test a non-default Cray PE with ReFrame: module load cdt/19.08 reframe --non-default-craype -r + + +Since CDT 19.11 you can load the CDT module from within ReFrame as follows: + +.. code:: bash + + reframe -m cdt/20.03 --non-default-craype -r diff --git a/reframe/__init__.py b/reframe/__init__.py index 3b9b04dc0b..82eca0cbf3 100644 --- a/reframe/__init__.py +++ b/reframe/__init__.py @@ -17,6 +17,8 @@ 'Python >= %d.%d.%d is required\n' % MIN_PYTHON_VERSION) sys.exit(1) +os.environ['RFM_INSTALL_PREFIX'] = INSTALL_PREFIX + # Import important names for user tests from reframe.core.pipeline import * # noqa: F401, F403 diff --git a/reframe/core/__init__.py b/reframe/core/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/reframe/core/backends.py b/reframe/core/backends.py new file mode 100644 index 0000000000..30ec9e90ad --- /dev/null +++ b/reframe/core/backends.py @@ -0,0 +1,62 @@ +# Copyright 2016-2020 Swiss National Supercomputing Centre (CSCS/ETH Zurich) +# ReFrame Project Developers. See the top-level LICENSE file for details. +# +# SPDX-License-Identifier: BSD-3-Clause + + +import importlib +import functools + +import reframe.core.fields as fields +from reframe.core.exceptions import ConfigError + +_launcher_backend_modules = [ + 'reframe.core.launchers.local', + 'reframe.core.launchers.mpi', + 'reframe.core.launchers.ssh' +] +_launchers = {} +_scheduler_backend_modules = [ + 'reframe.core.schedulers.local', + 'reframe.core.schedulers.slurm', + 'reframe.core.schedulers.pbs', + 'reframe.core.schedulers.torque' +] +_schedulers = {} + + +def _register_backend(name, local=False, *, backend_type): + def do_register(cls): + registry = globals()[f'_{backend_type}s'] + if name in registry: + raise ConfigError( + f"'{name}' is already registered as a {backend_type}" + ) + + cls.is_local = fields.ConstantField(bool(local)) + cls.registered_name = fields.ConstantField(name) + registry[name] = cls + return cls + + return do_register + + +def _get_backend(name, *, backend_type): + backend_modules = globals()[f'_{backend_type}_backend_modules'] + for mod in backend_modules: + importlib.import_module(mod) + + try: + return globals()[f'_{backend_type}s'][name] + except KeyError: + raise ConfigError(f"no such {backend_type}: '{name}'") + + +register_scheduler = functools.partial( + _register_backend, backend_type='scheduler' +) +register_launcher = functools.partial( + _register_backend, backend_type='launcher' +) +getscheduler = functools.partial(_get_backend, backend_type='scheduler') +getlauncher = functools.partial(_get_backend, backend_type='launcher') diff --git a/reframe/core/buildsystems.py b/reframe/core/buildsystems.py index 991d996a5b..74b45f55e2 100644 --- a/reframe/core/buildsystems.py +++ b/reframe/core/buildsystems.py @@ -111,11 +111,11 @@ def __init__(self): self.cxx = None self.ftn = None self.nvcc = None - self.cflags = None - self.cxxflags = None - self.cppflags = None - self.fflags = None - self.ldflags = None + self.cflags = [] + self.cxxflags = [] + self.cppflags = [] + self.fflags = [] + self.ldflags = [] self.flags_from_environ = True @abc.abstractmethod @@ -138,7 +138,7 @@ def emit_build_commands(self, environ): def _resolve_flags(self, flags, environ): _flags = getattr(self, flags) - if _flags is not None: + if _flags: return _flags if self.flags_from_environ: @@ -259,31 +259,31 @@ def emit_build_commands(self, environ): cxxflags = self._cxxflags(environ) fflags = self._fflags(environ) ldflags = self._ldflags(environ) - if cc is not None: + if cc: cmd_parts += ['CC="%s"' % cc] - if cxx is not None: + if cxx: cmd_parts += ['CXX="%s"' % cxx] - if ftn is not None: + if ftn: cmd_parts += ['FC="%s"' % ftn] - if nvcc is not None: + if nvcc: cmd_parts += ['NVCC="%s"' % nvcc] - if cppflags is not None: + if cppflags: cmd_parts += ['CPPFLAGS="%s"' % ' '.join(cppflags)] - if cflags is not None: + if cflags: cmd_parts += ['CFLAGS="%s"' % ' '.join(cflags)] - if cxxflags is not None: + if cxxflags: cmd_parts += ['CXXFLAGS="%s"' % ' '.join(cxxflags)] - if fflags is not None: + if fflags: cmd_parts += ['FCFLAGS="%s"' % ' '.join(fflags)] - if ldflags is not None: + if ldflags: cmd_parts += ['LDFLAGS="%s"' % ' '.join(ldflags)] if self.options: @@ -398,28 +398,28 @@ def emit_build_commands(self, environ): lang = self.lang or self._guess_language(self.srcfile) cmd_parts = [] if lang == 'C': - if cc is None: + if not cc: raise BuildSystemError('I do not know how to compile a ' 'C program') cmd_parts += [cc, *cppflags, *cflags, self.srcfile, '-o', executable, *ldflags] elif lang == 'C++': - if cxx is None: + if not cxx: raise BuildSystemError('I do not know how to compile a ' 'C++ program') cmd_parts += [cxx, *cppflags, *cxxflags, self.srcfile, '-o', executable, *ldflags] elif lang == 'Fortran': - if ftn is None: + if not ftn: raise BuildSystemError('I do not know how to compile a ' 'Fortran program') cmd_parts += [ftn, *cppflags, *fflags, self.srcfile, '-o', executable, *ldflags] elif lang == 'CUDA': - if nvcc is None: + if not nvcc: raise BuildSystemError('I do not know how to compile a ' 'CUDA program') @@ -506,7 +506,7 @@ class CMake(ConfigureBasedBuildSystem): ''' def _combine_flags(self, cppflags, xflags): - if cppflags is None: + if not cppflags: return xflags ret = list(cppflags) @@ -534,28 +534,28 @@ def emit_build_commands(self, environ): cxxflags = self._combine_flags(cppflags, self._cxxflags(environ)) fflags = self._combine_flags(cppflags, self._fflags(environ)) ldflags = self._ldflags(environ) - if cc is not None: + if cc: cmake_cmd += ['-DCMAKE_C_COMPILER="%s"' % cc] - if cxx is not None: + if cxx: cmake_cmd += ['-DCMAKE_CXX_COMPILER="%s"' % cxx] - if ftn is not None: + if ftn: cmake_cmd += ['-DCMAKE_Fortran_COMPILER="%s"' % ftn] - if nvcc is not None: + if nvcc: cmake_cmd += ['-DCMAKE_CUDA_COMPILER="%s"' % nvcc] - if cflags is not None: + if cflags: cmake_cmd += ['-DCMAKE_C_FLAGS="%s"' % ' '.join(cflags)] - if cxxflags is not None: + if cxxflags: cmake_cmd += ['-DCMAKE_CXX_FLAGS="%s"' % ' '.join(cxxflags)] - if fflags is not None: + if fflags: cmake_cmd += ['-DCMAKE_Fortran_FLAGS="%s"' % ' '.join(fflags)] - if ldflags is not None: + if ldflags: cmake_cmd += ['-DCMAKE_EXE_LINKER_FLAGS="%s"' % ' '.join(ldflags)] if self.config_opts: @@ -611,28 +611,28 @@ def emit_build_commands(self, environ): cxxflags = self._cxxflags(environ) fflags = self._fflags(environ) ldflags = self._ldflags(environ) - if cc is not None: + if cc: configure_cmd += ['CC="%s"' % cc] - if cxx is not None: + if cxx: configure_cmd += ['CXX="%s"' % cxx] - if ftn is not None: + if ftn: configure_cmd += ['FC="%s"' % ftn] - if cppflags is not None: + if cppflags: configure_cmd += ['CPPFLAGS="%s"' % ' '.join(cppflags)] - if cflags is not None: + if cflags: configure_cmd += ['CFLAGS="%s"' % ' '.join(cflags)] - if cxxflags is not None: + if cxxflags: configure_cmd += ['CXXFLAGS="%s"' % ' '.join(cxxflags)] - if fflags is not None: + if fflags: configure_cmd += ['FCFLAGS="%s"' % ' '.join(fflags)] - if ldflags is not None: + if ldflags: configure_cmd += ['LDFLAGS="%s"' % ' '.join(ldflags)] if self.config_opts: diff --git a/reframe/core/config.py b/reframe/core/config.py index 69eb8001ee..8cad64f281 100644 --- a/reframe/core/config.py +++ b/reframe/core/config.py @@ -3,256 +3,411 @@ # # SPDX-License-Identifier: BSD-3-Clause -import collections.abc +import copy +import fnmatch +import itertools import json import jsonschema import os import re +import socket import tempfile import reframe import reframe.core.debug as debug import reframe.core.fields as fields +import reframe.core.settings as settings import reframe.utility as util import reframe.utility.os_ext as os_ext import reframe.utility.typecheck as types -from reframe.core.exceptions import (ConfigError, ReframeError, +from reframe.core.exceptions import (ConfigError, + ReframeDeprecationWarning, ReframeFatalError) +from reframe.core.logging import getlogger +from reframe.utility import ScopedDict -_settings = None +def _match_option(opt, opt_map): + if isinstance(opt, list): + opt = '/'.join(opt) + if opt in opt_map: + return opt_map[opt] -def load_settings_from_file(filename): - global _settings - try: - _settings = util.import_module_from_file(filename).settings - return _settings - except Exception as e: - raise ConfigError( - "could not load configuration file `%s'" % filename) from e + for k, v in opt_map.items(): + if fnmatch.fnmatchcase(opt, k): + return v + raise KeyError(opt) -def settings(): - if _settings is None: - raise ReframeFatalError('ReFrame is not configured') - return _settings +class _SiteConfig: + def __init__(self, site_config, filename): + self._site_config = copy.deepcopy(site_config) + self._filename = filename + self._local_config = {} + self._local_system = None + self._sticky_options = {} + # Open and store the JSON schema for later validation + schema_filename = os.path.join(reframe.INSTALL_PREFIX, + 'schemas', 'config.json') + with open(schema_filename) as fp: + try: + self._schema = json.loads(fp.read()) + except json.JSONDecodeError as e: + raise ReframeFatalError( + f"invalid configuration schema: '{schema_filename}'" + ) from e -class SiteConfiguration: - '''Holds the configuration of systems and environments''' - _modes = fields.ScopedDictField('_modes', types.List[str]) - - def __init__(self, dict_config=None): - self._systems = {} - self._modes = {} - if dict_config is not None: - self.load_from_dict(dict_config) + def _pick_config(self): + return self._local_config if self._local_config else self._site_config def __repr__(self): - return debug.repr(self) + return (f'{type(self).__name__}(site_config={self._site_config!r}, ' + 'filename={self._filename!r})') - @property - def systems(self): - return self._systems + def __str__(self): + return json.dumps(self._pick_config(), indent=2) - @property - def modes(self): - return self._modes + # Delegate everything to either the original config or to the reduced one + # if a system is selected - def get_schedsystem_config(self, descr): - # Handle the special shortcuts first - from reframe.core.launchers.registry import getlauncher - from reframe.core.schedulers.registry import getscheduler + def __iter__(self): + return iter(self._pick_config()) - if descr == 'nativeslurm': - return getscheduler('slurm'), getlauncher('srun') + def __getitem__(self, key): + return self._pick_config()[key] - if descr == 'local': - return getscheduler('local'), getlauncher('local') + def __getattr__(self, attr): + return getattr(self._pick_config(), attr) - try: - sched_descr, launcher_descr = descr.split('+') - except ValueError: - raise ValueError('invalid syntax for the ' - 'scheduling system: %s' % descr) from None + def add_sticky_option(self, option, value): + self._sticky_options[option] = value - return getscheduler(sched_descr), getlauncher(launcher_descr) + def remove_sticky_option(self, option): + self._sticky_options.pop(option, None) - def load_from_dict(self, site_config): - if not isinstance(site_config, collections.abc.Mapping): - raise TypeError('site configuration is not a dict') + def get(self, option, default=None): + '''Retrieve value of option. - # We do all the necessary imports here and not on the top, because we - # want to remove import time dependencies - import reframe.core.environments as m_env - from reframe.core.systems import System, SystemPartition + If the option cannot be retrieved, ``default`` will be returned. + ''' - sysconfig = site_config.get('systems', None) - envconfig = site_config.get('environments', None) - modes = site_config.get('modes', {}) + # Options may not start with a slash + if not option or option[0] == '/': + return default - if not sysconfig: - raise ValueError('no entry for systems was found') + # Remove trailing / + if option[-1] == '/': + option = option[:-1] - if not envconfig: - raise ValueError('no entry for environments was found') + # Convert any indices to integers + prepared_option = [] + for opt in option.split('/'): + try: + opt = int(opt) + except ValueError: + pass + + prepared_option.append(opt) + + # Walk through the option path constructing a default key at the same + # time for looking it up in the defaults or the sticky options + default_key = [] + value = self._pick_config() + option_path_invalid = False + for x in prepared_option: + if option_path_invalid: + # Just go through the rest of elements and construct the key + # trivially + if not isinstance(x, int) and x[0] != '@': + default_key.append(x) + + continue + + if isinstance(x, int) or x[0] == '@': + # We are in an addressable element; move forward in the path, + # without adding the component to the default_key + if isinstance(x, int): + # Element addressable by index number + try: + value = value[x] + except IndexError: + option_path_invalid = True + else: + # Element addressable by name + x, found = x[1:], False + for obj in value: + value, found = obj, True + if obj['name'] == x: + value, found = obj, True + break - # Convert envconfig to a ScopedDict - try: - envconfig = fields.ScopedDict(envconfig) - except TypeError: - raise TypeError('environments configuration ' - 'is not a scoped dictionary') from None - - # Convert modes to a `ScopedDict`; note that `modes` will implicitly - # converted to a scoped dict here, since `self._modes` is a - # `ScopedDictField`. + if not found: + option_path_invalid = True + + continue + + if 'type' in value: + default_key.append(value['type'] + '_' + x) + else: + default_key.append(x) + + try: + value = value[x] + except (IndexError, KeyError, TypeError): + option_path_invalid = True + + default_key = '/'.join(default_key) try: - self._modes = modes - except TypeError: - raise TypeError('modes configuration ' - 'is not a scoped dictionary') from None + # If a sticky option exists, return that value + return _match_option(default_key, self._sticky_options) + except KeyError: + pass - def create_env(system, partition, name): - # Create an environment instance + if option_path_invalid: + # Try the default and return try: - config = envconfig['%s:%s:%s' % (system, partition, name)] + return _match_option(default_key, self._schema['defaults']) except KeyError: - raise ConfigError( - "could not find a definition for `%s'" % name - ) from None - - if not isinstance(config, collections.abc.Mapping): - raise TypeError("config for `%s' is not a dictionary" % name) - - return m_env.ProgEnvironment(name, **config) - - # Populate the systems directory - for sys_name, config in sysconfig.items(): - if not isinstance(config, dict): - raise TypeError('system configuration is not a dictionary') - - if not isinstance(config['partitions'], collections.abc.Mapping): - raise TypeError('partitions must be a dictionary') - - sys_descr = config.get('descr', sys_name) - sys_hostnames = config.get('hostnames', []) - - # The System's constructor provides also reasonable defaults, but - # since we are going to set them anyway from the values provided by - # the configuration, we should set default values here. The stage, - # output and log directories default to None, since they are going - # to be set dynamically by the runtime. - sys_prefix = config.get('prefix', '.') - sys_stagedir = config.get('stagedir', None) - sys_outputdir = config.get('outputdir', None) - sys_perflogdir = config.get('perflogdir', None) - sys_resourcesdir = config.get('resourcesdir', '.') - sys_modules_system = config.get('modules_system', None) - - # Expand variables - if sys_prefix: - sys_prefix = os_ext.expandvars(sys_prefix) - - if sys_stagedir: - sys_stagedir = os_ext.expandvars(sys_stagedir) - - if sys_outputdir: - sys_outputdir = os_ext.expandvars(sys_outputdir) - - if sys_perflogdir: - sys_perflogdir = os_ext.expandvars(sys_perflogdir) - - if sys_resourcesdir: - sys_resourcesdir = os_ext.expandvars(sys_resourcesdir) - - # Create the preload environment for the system - sys_preload_env = m_env.Environment( - name='__rfm_env_%s' % sys_name, - modules=config.get('modules', []), - variables=config.get('variables', {}) + return default + + return value + + @property + def filename(self): + return self._filename + + @property + def subconfig_system(self): + return self._local_system + + @classmethod + def create(cls, filename): + _, ext = os.path.splitext(filename) + if ext == '.py': + return cls._create_from_python(filename) + elif ext == '.json': + return cls._create_from_json(filename) + else: + raise ConfigError(f"unknown configuration file type: '{filename}'") + + @classmethod + def _create_from_python(cls, filename): + try: + mod = util.import_module_from_file(filename) + except ImportError as e: + # import_module_from_file() may raise an ImportError if the + # configuration file is under ReFrame's top-level directory + raise ConfigError( + f"could not load Python configuration file: '{filename}'" + ) from e + + if hasattr(mod, 'settings'): + # Looks like an old style config + raise ReframeDeprecationWarning( + f"the syntax of the configuration file '{filename}' " + f"is deprecated" ) - system = System(name=sys_name, - descr=sys_descr, - hostnames=sys_hostnames, - preload_env=sys_preload_env, - prefix=sys_prefix, - stagedir=sys_stagedir, - outputdir=sys_outputdir, - perflogdir=sys_perflogdir, - resourcesdir=sys_resourcesdir, - modules_system=sys_modules_system) - for part_name, partconfig in config.get('partitions', {}).items(): - if not isinstance(partconfig, collections.abc.Mapping): - raise TypeError("partition `%s' not configured " - "as a dictionary" % part_name) - - part_descr = partconfig.get('descr', part_name) - part_scheduler, part_launcher = self.get_schedsystem_config( - partconfig.get('scheduler', 'local+local') - ) - part_local_env = m_env.Environment( - name='__rfm_env_%s' % part_name, - modules=partconfig.get('modules', []), - variables=partconfig.get('variables', {}).items() - ) - part_environs = [ - create_env(sys_name, part_name, e) - for e in partconfig.get('environs', []) - ] - part_access = partconfig.get('access', []) - part_resources = partconfig.get('resources', {}) - part_max_jobs = partconfig.get('max_jobs', 1) - part = SystemPartition(name=part_name, - descr=part_descr, - scheduler=part_scheduler, - launcher=part_launcher, - access=part_access, - environs=part_environs, - resources=part_resources, - local_env=part_local_env, - max_jobs=part_max_jobs) - - container_platforms = partconfig.get('container_platforms', {}) - for cp, env_spec in container_platforms.items(): - cp_env = m_env.Environment( - name='__rfm_env_%s' % cp, - modules=env_spec.get('modules', []), - variables=env_spec.get('variables', {}) + mod = util.import_module_from_file(filename) + if not hasattr(mod, 'site_configuration'): + raise ConfigError( + f"not a valid Python configuration file: '{filename}'" + ) + + return _SiteConfig(mod.site_configuration, filename) + + @classmethod + def _create_from_json(cls, filename): + with open(filename) as fp: + try: + config = json.loads(fp.read()) + except json.JSONDecodeError as e: + raise ConfigError( + f"invalid JSON syntax in configuration file '{filename}'" + ) from e + + return _SiteConfig(config, filename) + + def _detect_system(self): + if os.path.exists('/etc/xthostname'): + # Get the cluster name on Cray systems + with open('/etc/xthostname') as fp: + hostname = fp.read() + else: + hostname = socket.gethostname() + + for system in self._site_config['systems']: + for patt in system['hostnames']: + if re.match(patt, hostname): + return system['name'] + + raise ConfigError(f"could not find a configuration entry " + f"for the current system: '{hostname}'") + + def validate(self): + site_config = self._pick_config() + try: + jsonschema.validate(site_config, self._schema) + except jsonschema.ValidationError as e: + raise ConfigError(f"could not validate configuration file: " + f"'{self._filename}'") from e + + # Make sure that system and partition names are unique + system_names = set() + for system in self._site_config['systems']: + sysname = system['name'] + if sysname in system_names: + raise ConfigError(f"system '{sysname}' already defined") + + system_names.add(sysname) + partition_names = set() + for part in system['partitions']: + partname = part['name'] + if partname in partition_names: + raise ConfigError( + f"partition '{partname}' already defined " + f"for system '{sysname}'" ) - part.add_container_env(cp, cp_env) - system.add_partition(part) + partition_names.add(partname) + + def select_subconfig(self, system_fullname=None): + if (self._local_system is not None and + self._local_system == system_fullname): + return + + system_fullname = system_fullname or self._detect_system() + try: + system_name, part_name = system_fullname.split(':', maxsplit=1) + except ValueError: + # system_name does not have a partition + system_name, part_name = system_fullname, None + + # Start from a fresh copy of the site_config, because we will be + # modifying it + site_config = copy.deepcopy(self._site_config) + self._local_config = {} + systems = list( + filter(lambda x: x['name'] == system_name, site_config['systems']) + ) + if not systems: + raise ConfigError( + f"could not find a configuration entry " + f"for the requested system: '{system_name}'" + ) + + if part_name is not None: + # Filter out also partitions + systems[0]['partitions'] = list( + filter(lambda x: x['name'] == part_name, + systems[0]['partitions']) + ) + + if not systems[0]['partitions']: + raise ConfigError( + f"could not find a configuration entry " + f"for the requested system/partition combination: " + f"'{system_name}:{part_name}'" + ) + + # Create local configuration for the current or the requested system + self._local_config['systems'] = systems + for name, section in site_config.items(): + if name == 'systems': + # The systems sections has already been treated + continue + + # Convert section to a scoped dict that will handle correctly and + # transparently the system/partition resolution + scoped_section = ScopedDict() + for obj in section: + key = obj.get('name', name) + target_systems = obj.get( + 'target_systems', + _match_option(f'{name}/target_systems', + self._schema['defaults']) + ) + for t in target_systems: + scoped_section[f'{t}:{key}'] = obj + + unique_keys = set() + for obj in section: + key = obj.get('name', name) + if key in unique_keys: + continue + + unique_keys.add(key) + try: + val = scoped_section[f"{system_fullname}:{key}"] + except KeyError: + pass + else: + self._local_config.setdefault(name, []) + self._local_config[name].append(val) + + required_sections = self._schema['required'] + for name in required_sections: + if name not in self._local_config.keys(): + raise ConfigError(f"section '{name}' not defined " + f"for system '{system_fullname}'") + + # Verify that all environments defined by the system are defined for + # the current system + sys_environs = { + *itertools.chain(*(p['environs'] + for p in systems[0]['partitions'])) + } + found_environs = { + e['name'] for e in self._local_config['environments'] + } + undefined_environs = sys_environs - found_environs + if undefined_environs: + env_descr = ', '.join(f"'{e}'" for e in undefined_environs) + raise ConfigError( + f"environments {env_descr} " + f"are not defined for '{system_fullname}'" + ) - self._systems[sys_name] = system + self._local_system = system_fullname def convert_old_config(filename): - old_config = load_settings_from_file(filename) + old_config = util.import_module_from_file(filename).settings converted = { 'systems': [], 'environments': [], 'logging': [], - 'perf_logging': [], } + perflogdir = None old_systems = old_config.site_configuration['systems'].items() - for sys_name, sys_specs in old_systems: + for sys_name, sys_spec in old_systems: sys_dict = {'name': sys_name} - sys_dict.update(sys_specs) + + # FIXME: We pick the perflogdir that we first find we use it as + # filelog's basedir for all systems. This is not correct since + # per-system customizations of perflogdir will be lost + if perflogdir is None: + perflogdir = sys_spec.pop('perflogdir', None) + + sys_dict.update(sys_spec) + + # hostnames is now a required property + if 'hostnames' not in sys_spec: + sys_dict['hostnames'] = [] # Make variables dictionary into a list of lists - if 'variables' in sys_specs: + if 'variables' in sys_spec: sys_dict['variables'] = [ [vname, v] for vname, v in sys_dict['variables'].items() ] # Make partitions dictionary into a list - if 'partitions' in sys_specs: + if 'partitions' in sys_spec: sys_dict['partitions'] = [] - for pname, p in sys_specs['partitions'].items(): + for pname, p in sys_spec['partitions'].items(): new_p = {'name': pname} new_p.update(p) if p['scheduler'] == 'nativeslurm': @@ -262,9 +417,9 @@ def convert_old_config(filename): new_p['scheduler'] = 'local' new_p['launcher'] = 'local' else: - sched, launch, *_ = p['scheduler'].split('+') + sched, launcher, *_ = p['scheduler'].split('+') new_p['scheduler'] = sched - new_p['launcher'] = launch + new_p['launcher'] = launcher # Make resources dictionary into a list if 'resources' in p: @@ -282,7 +437,7 @@ def convert_old_config(filename): if 'container_platforms' in p: new_p['container_platforms'] = [] for cname, c in p['container_platforms'].items(): - new_c = {'name': cname} + new_c = {'type': cname} new_c.update(c) if 'variables' in c: new_c['variables'] = [ @@ -327,22 +482,34 @@ def convert_old_config(filename): converted['modes'].append(new_mode) - def update_logging_config(log_name, original_log): - new_handlers = [] - for h in original_log['handlers']: + def handler_list(handler_config): + ret = [] + for h in handler_config: new_h = h new_h['level'] = h['level'].lower() - new_handlers.append(new_h) - - converted[log_name].append( - { - 'level': original_log['level'].lower(), - 'handlers': new_handlers - } - ) - - update_logging_config('logging', old_config.logging_config) - update_logging_config('perf_logging', old_config.perf_logging_config) + if h['type'] == 'graylog': + # `host` and `port` attribute are converted to `address` + new_h['address'] = h['host'] + if 'port' in h: + new_h['address'] += ':' + h['port'] + elif h['type'] == 'filelog' and perflogdir is not None: + new_h['basedir'] = perflogdir + + ret.append(new_h) + + return ret + + converted['logging'].append( + { + 'level': old_config.logging_config['level'].lower(), + 'handlers': handler_list( + old_config.logging_config['handlers'] + ), + 'handlers_perflog': handler_list( + old_config.perf_logging_config['handlers'] + ) + } + ) converted['general'] = [{}] if hasattr(old_config, 'checks_path'): converted['general'][0][ @@ -357,18 +524,39 @@ def update_logging_config(log_name, original_log): if converted['general'] == [{}]: del converted['general'] - # Validate the converted file - schema_filename = os.path.join(reframe.INSTALL_PREFIX, - 'schemas', 'config.json') - - # We let the following statements raise, because if they do, that's a BUG - with open(schema_filename) as fp: - schema = json.loads(fp.read()) - - jsonschema.validate(converted, schema) - with tempfile.NamedTemporaryFile(mode='w', delete=False) as fp: + with tempfile.NamedTemporaryFile(mode='w', suffix='.py', + delete=False) as fp: fp.write(f"#\n# This file was automatically generated " f"by ReFrame based on '{filename}'.\n#\n\n") fp.write(f'site_configuration = {util.ppretty(converted)}\n') return fp.name + + +def _find_config_file(): + # The order of elements is important, since it defines the priority + prefixes = [ + os.path.join(os.getlogin(), '.reframe'), + reframe.INSTALL_PREFIX, + '/etc/reframe.d' + ] + valid_exts = ['py', 'json'] + for d in prefixes: + for ext in valid_exts: + filename = os.path.join(d, f'settings.{ext}') + if os.path.exists(filename): + return filename + + return None + + +def load_config(filename=None): + if filename is None: + filename = _find_config_file() + if filename is None: + # Return the generic configuration + getlogger().debug('no configuration found; ' + 'falling back to a generic one') + return _SiteConfig(settings.site_configuration, '') + + return _SiteConfig.create(filename) diff --git a/reframe/core/containers.py b/reframe/core/containers.py index 456dc2cc95..afdda0486f 100644 --- a/reframe/core/containers.py +++ b/reframe/core/containers.py @@ -150,9 +150,10 @@ def launch_command(self): ['cd ' + self.workdir] + self.commands) + "'" -class ShifterNG(Sarus): - '''Container platform backend for running containers with `ShifterNG - `__.''' +class Shifter(Sarus): + '''Container platform backend for running containers with `Shifter + `__. + ''' def __init__(self): super().__init__() diff --git a/reframe/core/environments.py b/reframe/core/environments.py index 0b5cc7449a..e8806fbc78 100644 --- a/reframe/core/environments.py +++ b/reframe/core/environments.py @@ -10,7 +10,6 @@ import reframe.utility as util import reframe.utility.os_ext as os_ext import reframe.utility.typecheck as typ -from reframe.core.runtime import runtime class Environment: @@ -19,11 +18,10 @@ class Environment: It is simply a collection of modules to be loaded and environment variables to be set when this environment is loaded by the framework. ''' - name = fields.TypedField('name', typ.Str[r'(\w|-)+']) - modules = fields.TypedField('modules', typ.List[str]) - variables = fields.TypedField('variables', typ.Dict[str, str]) - def __init__(self, name, modules=[], variables=[]): + def __init__(self, name, modules=None, variables=None): + modules = modules or [] + variables = variables or [] self._name = name self._modules = list(modules) self._variables = collections.OrderedDict(variables) @@ -52,16 +50,6 @@ def variables(self): ''' return util.MappingView(self._variables) - @property - def is_loaded(self): - ''':class:`True` if this environment is loaded, - :class:`False` otherwise. - ''' - is_module_loaded = runtime().modules_system.is_module_loaded - return (all(map(is_module_loaded, self._modules)) and - all(os.environ.get(k, None) == os_ext.expandvars(v) - for k, v in self._variables.items())) - def details(self): '''Return a detailed description of this environment.''' variables = '\n'.join(' '*8 + '- %s=%s' % (k, v) @@ -85,16 +73,15 @@ def __str__(self): return self.name def __repr__(self): - ret = "{0}(name='{1}', modules={2}, variables={3})" - return ret.format(type(self).__name__, self.name, - self.modules, self.variables) + return (f'{type(self).__name__}(' + f'name={self._name!r}, ' + f'modules={self._modules!r}, ' + f'variables={list(self._variables.items())!r})') class _EnvironmentSnapshot(Environment): def __init__(self, name='env_snapshot'): - super().__init__(name, - runtime().modules_system.loaded_modules(), - os.environ.items()) + super().__init__(name, [], os.environ.items()) def restore(self): '''Restore this environment snapshot.''' @@ -110,8 +97,7 @@ def __eq__(self, other): if other.variables[k] != v: return False - return (self.name == other.name and - set(self.modules) == set(other.modules)) + return self.name == other.name def snapshot(): @@ -119,52 +105,6 @@ def snapshot(): return _EnvironmentSnapshot() -def load(*environs): - '''Load environments in the current Python context. - - Returns a tuple containing a snapshot of the environment at entry to this - function and a list of shell commands required to load ``environs``. - ''' - env_snapshot = snapshot() - commands = [] - rt = runtime() - for env in environs: - for m in env.modules: - conflicted = rt.modules_system.load_module(m, force=True) - for c in conflicted: - commands += rt.modules_system.emit_unload_commands(c) - - commands += rt.modules_system.emit_load_commands(m) - - for k, v in env.variables.items(): - os.environ[k] = os_ext.expandvars(v) - commands.append('export %s=%s' % (k, v)) - - return env_snapshot, commands - - -def emit_load_commands(*environs): - env_snapshot, commands = load(*environs) - env_snapshot.restore() - return commands - - -class temp_environment: - '''Context manager to temporarily change the environment.''' - - def __init__(self, modules=[], variables=[]): - self._modules = modules - self._variables = variables - - def __enter__(self): - new_env = Environment('_rfm_temp_env', self._modules, self._variables) - self._environ_save, _ = load(new_env) - return new_env - - def __exit__(self, exc_type, exc_value, traceback): - self._environ_save.restore() - - class ProgEnvironment(Environment): '''A class representing a programming environment. @@ -184,16 +124,16 @@ class ProgEnvironment(Environment): _cc = fields.TypedField('_cc', str) _cxx = fields.TypedField('_cxx', str) _ftn = fields.TypedField('_ftn', str) - _cppflags = fields.TypedField('_cppflags', typ.List[str], type(None)) - _cflags = fields.TypedField('_cflags', typ.List[str], type(None)) - _cxxflags = fields.TypedField('_cxxflags', typ.List[str], type(None)) - _fflags = fields.TypedField('_fflags', typ.List[str], type(None)) - _ldflags = fields.TypedField('_ldflags', typ.List[str], type(None)) + _cppflags = fields.TypedField('_cppflags', typ.List[str]) + _cflags = fields.TypedField('_cflags', typ.List[str]) + _cxxflags = fields.TypedField('_cxxflags', typ.List[str]) + _fflags = fields.TypedField('_fflags', typ.List[str]) + _ldflags = fields.TypedField('_ldflags', typ.List[str]) def __init__(self, name, - modules=[], - variables={}, + modules=None, + variables=None, cc='cc', cxx='CC', ftn='ftn', @@ -209,11 +149,11 @@ def __init__(self, self._cxx = cxx self._ftn = ftn self._nvcc = nvcc - self._cppflags = cppflags - self._cflags = cflags - self._cxxflags = cxxflags - self._fflags = fflags - self._ldflags = ldflags + self._cppflags = cppflags or [] + self._cflags = cflags or [] + self._cxxflags = cxxflags or [] + self._fflags = fflags or [] + self._ldflags = ldflags or [] @property def cc(self): @@ -227,7 +167,7 @@ def cc(self): def cxx(self): '''The C++ compiler of this programming environment. - :type: :class:`str` or :class:`None` + :type: :class:`str` ''' return self._cxx @@ -235,7 +175,7 @@ def cxx(self): def ftn(self): '''The Fortran compiler of this programming environment. - :type: :class:`str` or :class:`None` + :type: :class:`str` ''' return self._ftn @@ -243,7 +183,7 @@ def ftn(self): def cppflags(self): '''The preprocessor flags of this programming environment. - :type: :class:`str` or :class:`None` + :type: :class:`List[str]` ''' return self._cppflags @@ -251,7 +191,7 @@ def cppflags(self): def cflags(self): '''The C compiler flags of this programming environment. - :type: :class:`str` or :class:`None` + :type: :class:`List[str]` ''' return self._cflags @@ -259,7 +199,7 @@ def cflags(self): def cxxflags(self): '''The C++ compiler flags of this programming environment. - :type: :class:`str` or :class:`None` + :type: :class:`List[str]` ''' return self._cxxflags @@ -267,7 +207,7 @@ def cxxflags(self): def fflags(self): '''The Fortran compiler flags of this programming environment. - :type: :class:`str` or :class:`None` + :type: :class:`List[str]` ''' return self._fflags @@ -275,7 +215,7 @@ def fflags(self): def ldflags(self): '''The linker flags of this programming environment. - :type: :class:`str` or :class:`None` + :type: :class:`List[str]` ''' return self._ldflags @@ -285,10 +225,8 @@ def nvcc(self): def details(self): def format_flags(flags): - if flags is None: + if not flags: return '' - elif len(flags) == 0: - return "''" else: return ' '.join(flags) diff --git a/reframe/core/launchers/__init__.py b/reframe/core/launchers/__init__.py index 07d223738b..ecbae3d284 100644 --- a/reframe/core/launchers/__init__.py +++ b/reframe/core/launchers/__init__.py @@ -19,13 +19,16 @@ class JobLauncher(abc.ABC): Users cannot create job launchers directly. You may retrieve a registered launcher backend through the - :func:`reframe.core.launchers.registry.getlauncher` function. + :func:`reframe.core.backends.getlauncher` function. .. note:: .. versionchanged:: 2.8 Job launchers do not get a reference to a job during their initialization. + .. note:: + .. versionchanged:: 3.0 + The :func:`getlauncher` function has moved to a different module. ''' #: List of options to be passed to the job launcher invocation. diff --git a/reframe/core/launchers/local.py b/reframe/core/launchers/local.py index 0b48bf95e0..0174667acb 100644 --- a/reframe/core/launchers/local.py +++ b/reframe/core/launchers/local.py @@ -3,10 +3,9 @@ # # SPDX-License-Identifier: BSD-3-Clause +from reframe.core.backends import register_launcher from reframe.core.launchers import JobLauncher -from reframe.core.launchers.registry import register_launcher - @register_launcher('local', local=True) class LocalLauncher(JobLauncher): diff --git a/reframe/core/launchers/mpi.py b/reframe/core/launchers/mpi.py index 55f5facb28..dca6e95129 100644 --- a/reframe/core/launchers/mpi.py +++ b/reframe/core/launchers/mpi.py @@ -3,8 +3,8 @@ # # SPDX-License-Identifier: BSD-3-Clause +from reframe.core.backends import register_launcher from reframe.core.launchers import JobLauncher -from reframe.core.launchers.registry import register_launcher from reframe.utility import seconds_to_hms diff --git a/reframe/core/launchers/registry.py b/reframe/core/launchers/registry.py deleted file mode 100644 index cae6f15f93..0000000000 --- a/reframe/core/launchers/registry.py +++ /dev/null @@ -1,87 +0,0 @@ -# Copyright 2016-2020 Swiss National Supercomputing Centre (CSCS/ETH Zurich) -# ReFrame Project Developers. See the top-level LICENSE file for details. -# -# SPDX-License-Identifier: BSD-3-Clause - -import reframe.core.fields as fields -from reframe.core.exceptions import ConfigError - - -# Name registry for job launchers -_LAUNCHERS = {} - - -def register_launcher(name, local=False): - '''Class decorator for registering new job launchers. - - .. caution:: - This decorator is only relevant to developers of new job launchers. - - .. note:: - .. versionadded:: 2.8 - - :arg name: The registration name of this launcher - :arg local: :class:`True` if launcher may only submit local jobs, - :class:`False` otherwise. - :raises ValueError: if a job launcher is already registered with - the same name. - ''' - - # See reference.rst for documentation - def _register_launcher(cls): - if name in _LAUNCHERS: - raise ValueError("a job launcher is already " - "registered with name '%s'" % name) - - cls.is_local = fields.ConstantField(bool(local)) - cls.registered_name = fields.ConstantField(name) - _LAUNCHERS[name] = cls - return cls - - return _register_launcher - - -def getlauncher(name): - '''Get launcher by its registered name. - - The available names are those specified in the - :doc:`configuration file `. - - This method may become handy in very special situations, e.g., testing an - application that needs to replace the system partition launcher or if a - different launcher must be used for a different programming environment. - - For example, if you want to replace the current partition's launcher with - the local one, here is how you can achieve it: - - :: - - def setup(self, partition, environ, **job_opts): - super().setup(partition, environ, **job_opts) - self.job.launcher = getlauncher('local')() - - - Note that this method returns a launcher class type and not an instance of - that class. - You have to instantiate it explicitly before assigning it to the - :attr:`launcher` attribute of the job. - - .. note:: - .. versionadded:: 2.8 - - :arg name: The name of the launcher to retrieve. - :returns: The class of the launcher requested, which is a subclass of - :class:`reframe.core.launchers.JobLauncher`. - :raises reframe.core.exceptions.ConfigError: if no launcher is - registered with that name. - ''' - try: - return _LAUNCHERS[name] - except KeyError: - raise ConfigError("no such job launcher: '%s'" % name) - - -# Import the launchers modules to trigger their registration -import reframe.core.launchers.local # noqa: F401, F403 -import reframe.core.launchers.mpi # noqa: F401, F403 -import reframe.core.launchers.ssh # noqa: F401, F403 diff --git a/reframe/core/launchers/ssh.py b/reframe/core/launchers/ssh.py index eb88e2bb98..79745f20ef 100644 --- a/reframe/core/launchers/ssh.py +++ b/reframe/core/launchers/ssh.py @@ -3,8 +3,8 @@ # # SPDX-License-Identifier: BSD-3-Clause +from reframe.core.backends import register_launcher from reframe.core.launchers import JobLauncher -from reframe.core.launchers.registry import register_launcher @register_launcher('ssh') diff --git a/reframe/core/logging.py b/reframe/core/logging.py index b8b1aa9fb1..b03d8006ed 100644 --- a/reframe/core/logging.py +++ b/reframe/core/logging.py @@ -184,74 +184,38 @@ def format(self, record): return super().format(record) -def load_from_dict(logging_config): - if not isinstance(logging_config, collections.abc.Mapping): - raise TypeError('logging configuration is not a dict') - - level = logging_config.get('level', 'info').lower() - handlers_descr = logging_config.get('handlers', None) +def _create_logger(site_config, handlers_group): + level = site_config.get('logging/0/level') logger = Logger('reframe') logger.setLevel(_log_level_values[level]) - for handler in _extract_handlers(handlers_descr): + for handler in _extract_handlers(site_config, handlers_group): logger.addHandler(handler) return logger -def _convert_handler_syntax(handler_dict): - handler_list = [] - for filename, handler_config in handler_dict.items(): - descr = handler_config - new_keys = {} - if filename == '&1': - new_keys['type'] = 'stream' - new_keys['name'] = 'stdout' - elif filename == '&2': - new_keys['type'] = 'stream' - new_keys['name'] = 'stderr' - else: - new_keys['type'] = 'file' - new_keys['name'] = filename - - descr.update(new_keys) - handler_list.append(descr) - - return handler_list - - -def _create_file_handler(handler_config): - try: - filename = handler_config['name'] - except KeyError: - raise ConfigError('no file specified for file handler:\n%s' % - pprint.pformat(handler_config)) from None - - timestamp = handler_config.get('timestamp', None) +def _create_file_handler(site_config, config_prefix): + filename = site_config.get(f'{config_prefix}/name') + timestamp = site_config.get(f'{config_prefix}/timestamp') if timestamp: basename, ext = os.path.splitext(filename) filename = '%s_%s%s' % (basename, time.strftime(timestamp), ext) - append = handler_config.get('append', False) + append = site_config.get(f'{config_prefix}/append') return logging.handlers.RotatingFileHandler(filename, mode='a+' if append else 'w+') -def _create_filelog_handler(handler_config): - logdir = os.path.abspath(LOG_CONFIG_OPTS['handlers.filelog.prefix']) - try: - filename_patt = os.path.join(logdir, handler_config['prefix']) - except KeyError: - raise ConfigError('no file specified for file handler:\n%s' % - pprint.pformat(handler_config)) from None - - append = handler_config.get('append', False) +def _create_filelog_handler(site_config, config_prefix): + basedir = os.path.abspath(site_config.get(f'{config_prefix}/basedir')) + prefix = site_config.get(f'{config_prefix}/prefix') + filename_patt = os.path.join(basedir, prefix) + append = site_config.get(f'{config_prefix}/append') return MultiFileHandler(filename_patt, mode='a+' if append else 'w+') -def _create_syslog_handler(handler_config): - address = handler_config.get('address', None) - if address is None: - raise ConfigError('syslog handler: no address specified') +def _create_syslog_handler(site_config, config_prefix): + address = site_config.get(f'{config_prefix}/address') # Check if address is in `host:port` format try: @@ -261,101 +225,100 @@ def _create_syslog_handler(handler_config): else: address = (host, port) - facility = handler_config.get('facility', 'user') + facility = site_config.get(f'{config_prefix}/facility') try: facility_type = logging.handlers.SysLogHandler.facility_names[facility] except KeyError: - raise ConfigError('syslog handler: ' - 'unknown facility: %s' % facility) from None + # This should not happen + raise AssertionError( + f'syslog handler: unknown facility: {facility}') from None - socktype = handler_config.get('socktype', 'udp') + socktype = site_config.get(f'{config_prefix}/socktype') if socktype == 'udp': socket_type = socket.SOCK_DGRAM elif socktype == 'tcp': socket_type = socket.SOCK_STREAM else: - raise ConfigError('syslog handler: unknown socket type: %s' % socktype) + # This should not happen + raise AssertionError( + f'syslog handler: unknown socket type: {socktype}' + ) return logging.handlers.SysLogHandler(address, facility_type, socket_type) -def _create_stream_handler(handler_config): - stream = handler_config.get('name', 'stdout') +def _create_stream_handler(site_config, config_prefix): + stream = site_config.get(f'{config_prefix}/name') if stream == 'stdout': return logging.StreamHandler(stream=sys.stdout) elif stream == 'stderr': return logging.StreamHandler(stream=sys.stderr) else: - raise ConfigError('unknown stream: %s' % stream) + # This should not happen + raise AssertionError(f'unknown stream: {stream}') -def _create_graylog_handler(handler_config): +def _create_graylog_handler(site_config, config_prefix): try: import pygelf except ImportError: return None - host = handler_config.get('host', None) - port = handler_config.get('port', None) - extras = handler_config.get('extras', None) - if host is None: - raise ConfigError('graylog handler: no host specified') - - if port is None: + address = site_config.get(f'{config_prefix}/address') + host, *port = address.split(':', maxsplit=1) + if not port: raise ConfigError('graylog handler: no port specified') - if extras is not None and not isinstance(extras, collections.abc.Mapping): - raise ConfigError('graylog handler: extras must be a mapping type') + port = port[0] + + # Check if the remote server is up and accepts connections; if not we will + # skip the handler + try: + with socket.create_connection((host, port), timeout=1): + pass + except OSError as e: + getlogger().warning( + f"could not connect to Graylog server at '{address}': {e}" + ) + return None + extras = site_config.get(f'{config_prefix}/extras') return pygelf.GelfHttpHandler(host=host, port=port, debug=True, static_fields=extras, include_extra_fields=True) -def _extract_handlers(handlers_list): - # Check if we are using the old syntax - if isinstance(handlers_list, collections.abc.Mapping): - handlers_list = _convert_handler_syntax(handlers_list) - sys.stderr.write( - 'WARNING: looks like you are using an old syntax for the ' - 'logging configuration; please update your syntax as follows:\n' - '\nhandlers: %s\n' % pprint.pformat(handlers_list, indent=1)) - +def _extract_handlers(site_config, handlers_group): + handler_prefix = f'logging/0/{handlers_group}' + handlers_list = site_config.get(handler_prefix) handlers = [] - if not handlers_list: - raise ValueError('no handlers are defined for logging') - for i, handler_config in enumerate(handlers_list): - if not isinstance(handler_config, collections.abc.Mapping): - raise TypeError('handler config at position %s ' - 'is not a dictionary' % i) - - try: - handler_type = handler_config['type'] - except KeyError: - raise ConfigError('no type specified for ' - 'handler at position %s' % i) from None - + handler_type = handler_config['type'] if handler_type == 'file': - hdlr = _create_file_handler(handler_config) + hdlr = _create_file_handler(site_config, f'{handler_prefix}/{i}') elif handler_type == 'filelog': - hdlr = _create_filelog_handler(handler_config) + hdlr = _create_filelog_handler( + site_config, f'{handler_prefix}/{i}' + ) elif handler_type == 'syslog': - hdlr = _create_syslog_handler(handler_config) + hdlr = _create_syslog_handler(site_config, f'{handler_prefix}/{i}') elif handler_type == 'stream': - hdlr = _create_stream_handler(handler_config) + hdlr = _create_stream_handler(site_config, f'{handler_prefix}/{i}') elif handler_type == 'graylog': - hdlr = _create_graylog_handler(handler_config) + hdlr = _create_graylog_handler( + site_config, f'{handler_prefix}/{i}' + ) if hdlr is None: - sys.stderr.write('WARNING: could not initialize the ' - 'graylog handler; ignoring...\n') + getlogger().warning('could not initialize the ' + 'graylog handler; ignoring ...') continue else: - raise ConfigError('unknown handler type: %s' % handler_type) + # Should not enter here + raise AssertionError(f"unknown handler type: {handler_type}") - level = handler_config.get('level', 'debug').lower() - fmt = handler_config.get('format', '%(message)s') - datefmt = handler_config.get('datefmt', '%FT%T') + level = site_config.get(f'{handler_prefix}/{i}/level') + fmt = site_config.get(f'{handler_prefix}/{i}/format') + datefmt = site_config.get(f'{handler_prefix}/{i}/datefmt') hdlr.setFormatter(RFC3339Formatter(fmt=fmt, datefmt=datefmt)) hdlr.setLevel(_check_level(level)) handlers.append(hdlr) @@ -577,24 +540,19 @@ def __exit__(self, exc_type, exc_value, traceback): _context_logger = self._orig_logger -def configure_logging(loggin_config): - global _logger, _context_logger +def configure_logging(site_config): + global _logger, _context_logger, _perf_logger - if loggin_config is None: + if site_config is None: _logger = None _context_logger = null_logger return - _logger = load_from_dict(loggin_config) + _logger = _create_logger(site_config, 'handlers') + _perf_logger = _create_logger(site_config, 'handlers_perflog') _context_logger = LoggerAdapter(_logger) -def configure_perflogging(perf_logging_config): - global _perf_logger - - _perf_logger = load_from_dict(perf_logging_config) - - def save_log_files(dest): os.makedirs(dest, exist_ok=True) for hdlr in _logger.handlers: diff --git a/reframe/core/modules.py b/reframe/core/modules.py index b6da5cd887..c5bbd2bc70 100644 --- a/reframe/core/modules.py +++ b/reframe/core/modules.py @@ -89,7 +89,7 @@ class ModulesSystem: @classmethod def create(cls, modules_kind=None): - if modules_kind is None: + if modules_kind is None or modules_kind == 'nomod': return ModulesSystem(NoModImpl()) elif modules_kind == 'tmod31': return ModulesSystem(TMod31Impl()) @@ -493,7 +493,8 @@ def unload_all(self): self._exec_module_command('purge') def searchpath(self): - return os.environ['MODULEPATH'].split(':') + path = os.getenv('MODULEPATH', '') + return path.split(':') def searchpath_add(self, *dirs): self._exec_module_command('use', *dirs) diff --git a/reframe/core/pipeline.py b/reframe/core/pipeline.py index ed1f1e4bbb..3182250703 100644 --- a/reframe/core/pipeline.py +++ b/reframe/core/pipeline.py @@ -27,17 +27,15 @@ import reframe.utility.os_ext as os_ext import reframe.utility.sanity as sn import reframe.utility.typecheck as typ +from reframe.core.backends import (getlauncher, getscheduler) from reframe.core.buildsystems import BuildSystemField from reframe.core.containers import ContainerPlatform, ContainerPlatformField from reframe.core.deferrable import _DeferredExpression from reframe.core.exceptions import (BuildError, DependencyError, PipelineError, SanityError, PerformanceError) -from reframe.core.launchers.registry import getlauncher from reframe.core.meta import RegressionTestMeta from reframe.core.schedulers import Job -from reframe.core.schedulers.registry import getscheduler -from reframe.core.systems import SystemPartition # Dependency kinds @@ -124,27 +122,13 @@ class RegressionTest(metaclass=RegressionTestMeta): This class provides the implementation of the pipeline phases that the regression test goes through during its lifetime. - :arg name: The name of the test. - If :class:`None`, the framework will try to assign a unique and - human-readable name to the test. - - :arg prefix: The directory prefix of the test. - If :class:`None`, the framework will set it to the directory containing - the test file. - .. note:: - The ``name`` and ``prefix`` arguments are just maintained for backward - compatibility to the old (prior to 2.13) syntax of regression tests. - Users are advised to use the new simplified syntax for writing - regression tests. - Refer to the :doc:`ReFrame Tutorial ` for more information. + .. versionchanged:: 2.19 - This class is also directly available under the top-level - :mod:`reframe` module. - - .. versionchanged:: 2.13 + Base constructor takes no arguments. ''' + #: The name of the test. #: #: :type: string that can contain any character except ``/`` @@ -672,19 +656,6 @@ class RegressionTest(metaclass=RegressionTestMeta): extra_resources = fields.TypedField('extra_resources', typ.Dict[str, typ.Dict[str, object]]) - # Private properties - _prefix = fields.TypedField('_prefix', str) - _stagedir = fields.TypedField('_stagedir', str, type(None)) - _stdout = fields.TypedField('_stdout', str, type(None)) - _stderr = fields.TypedField('_stderr', str, type(None)) - _current_partition = fields.TypedField('_current_partition', - SystemPartition, type(None)) - _current_environ = fields.TypedField('_current_environ', - env.Environment, type(None)) - _cdt_environ = fields.TypedField('_cdt_environ', env.Environment) - _job = fields.TypedField('_job', Job, type(None)) - _build_job = fields.TypedField('_build_job', Job, type(None)) - def __new__(cls, *args, **kwargs): obj = super().__new__(cls) @@ -803,7 +774,7 @@ def _rfm_init(self, name=None, prefix=None): # Weak reference to the test case associated with this check self._case = None - if rt.runtime().non_default_craype: + if rt.runtime().get_option('general/0/non_default_craype'): self._cdt_environ = env.Environment( name='__rfm_cdt_environ', variables={ @@ -832,7 +803,7 @@ def current_partition(self): This is set by the framework during the :func:`setup` phase. - :type: :class:`reframe.core.systems.SystemPartition`. + :type: :class:`reframe.core.systems._SystemPartition`. ''' return self._current_partition @@ -937,12 +908,13 @@ def build_stderr(self): return self._build_job.stderr def info(self): - '''Provide live information of a running test. + '''Provide live information for this test. - This method is used by the front-end to print the status message during - the test's execution. - This function is also called to provide the message for the - ``check_info`` `logging attribute `__. + This method is used by the front-end to print the status message + during the test's execution. This function is also called to provide + the message for the `check_info + `__ logging + attribute. By default, it returns a message reporting the test name, the current partition and the current programming environment that the test is currently executing on. @@ -998,13 +970,15 @@ def _setup_paths(self): '''Setup the check's dynamic paths.''' self.logger.debug('setting up paths') try: - resources = rt.runtime().resources - self._stagedir = resources.make_stagedir( + runtime = rt.runtime() + self._stagedir = runtime.make_stagedir( self.current_system.name, self._current_partition.name, - self._current_environ.name, self.name) - self._outputdir = resources.make_outputdir( + self._current_environ.name, self.name + ) + self._outputdir = runtime.make_outputdir( self.current_system.name, self._current_partition.name, - self._current_environ.name, self.name) + self._current_environ.name, self.name + ) except OSError as e: raise PipelineError('failed to set up paths') from e diff --git a/reframe/core/runtime.py b/reframe/core/runtime.py index ae59fdd6a0..264d905325 100644 --- a/reframe/core/runtime.py +++ b/reframe/core/runtime.py @@ -9,114 +9,35 @@ import os import functools -import re -import socket from datetime import datetime import reframe.core.config as config import reframe.core.fields as fields import reframe.utility.os_ext as os_ext -from reframe.core.exceptions import (ConfigError, - ReframeFatalError, - SpawnedProcessError, - SystemAutodetectionError, - UnknownSystemError) -from reframe.core.modules import ModulesSystem +from reframe.core.environments import (Environment, snapshot) +from reframe.core.exceptions import ReframeFatalError +from reframe.core.systems import System -class HostSystem: - '''The host system of the framework. - - The host system is a representation of the system that the framework - currently runs on.If the framework is properly configured, the host - system is automatically detected. If not, it may be explicitly set by the - user. +class RuntimeContext: + '''The runtime context of the framework. - This class is mainly a proxy of :class:`reframe.core.systems.System` that - stores optionally a partition name and provides some additional - functionality for manipulating system partitions. + This class essentially groups the current host system and the associated + resources of the framework on the current system. + It also encapsulates other runtime parameters that are relevant to the + framework's execution. - All attributes of the :class:`reframe.core.systems.System` may be accessed - directly from this proxy. + There is a single instance of this class globally in the framework. .. note:: .. versionadded:: 2.13 - ''' - - def __init__(self, system, partname=None): - self._system = system - self._partname = partname - - def __getattr__(self, attr): - # Delegate any failed attribute lookup to our backend - return getattr(self._system, attr) - - @property - def partitions(self): - '''The partitions of this system. - - :type: :class:`list[reframe.core.systems.SystemPartition]`. - ''' - - if not self._partname: - return self._system.partitions - - return [p for p in self._system.partitions if p.name == self._partname] - - def partition(self, name): - '''Return the system partition ``name``. - :type: :class:`reframe.core.systems.SystemPartition`. - ''' - for p in self.partitions: - if p.name == name: - return p - - return None - - def __str__(self): - partitions = '\n'.join(re.sub('(?m)^', 6*' ', '- ' + str(p)) - for p in self.partitions) - lines = [ - '%s [%s]:' % (self._name, self._descr), - ' hostnames: ' + ', '.join(self._hostnames), - ' modules_system: ' + str(self._modules_system), - ' resourcesdir: ' + self._resourcesdir, - ' partitions:\n' + partitions, - ] - return '\n'.join(lines) - - def __repr__(self): - return 'HostSystem(%r, %r)' % (self._system, self._partname) - - -class HostResources: - '''Resources associated with ReFrame execution on the current host. - - .. note:: - .. versionadded:: 2.13 ''' - #: The prefix directory of ReFrame execution. - #: This is always an absolute path. - #: - #: :type: :class:`str` - #: - #: .. caution:: - #: Users may not set this field. - #: - prefix = fields.AbsolutePathField('prefix') - outputdir = fields.AbsolutePathField('outputdir', type(None)) - stagedir = fields.AbsolutePathField('stagedir', type(None)) - perflogdir = fields.AbsolutePathField('perflogdir', type(None)) - - def __init__(self, prefix=None, stagedir=None, - outputdir=None, perflogdir=None, timefmt=None): - self.prefix = prefix or '.' - self.stagedir = stagedir - self.outputdir = outputdir - self.perflogdir = perflogdir - self.timefmt = timefmt + def __init__(self, site_config): + self._site_config = site_config + self._system = System.create(site_config) + self._current_run = 0 self._timestamp = datetime.now() def _makedir(self, *dirs, wipeout=False): @@ -140,104 +61,6 @@ def _format_dirs(self, *dirs): last += '_retry%s' % current_run return (*dirs[:-1], last) - @property - def timestamp(self): - return self._timestamp.strftime(self.timefmt) if self.timefmt else '' - - @property - def output_prefix(self): - '''The output prefix directory of ReFrame.''' - if self.outputdir is None: - return os.path.join(self.prefix, 'output', self.timestamp) - else: - return os.path.join(self.outputdir, self.timestamp) - - @property - def stage_prefix(self): - '''The stage prefix directory of ReFrame.''' - if self.stagedir is None: - return os.path.join(self.prefix, 'stage', self.timestamp) - else: - return os.path.join(self.stagedir, self.timestamp) - - @property - def perflog_prefix(self): - if self.perflogdir is None: - return os.path.join(self.prefix, 'perflogs') - else: - return self.perflogdir - - def make_stagedir(self, *dirs, wipeout=True): - return self._makedir(self.stage_prefix, - *self._format_dirs(*dirs), wipeout=wipeout) - - def make_outputdir(self, *dirs, wipeout=True): - return self._makedir(self.output_prefix, - *self._format_dirs(*dirs), wipeout=wipeout) - - -class RuntimeContext: - '''The runtime context of the framework. - - This class essentially groups the current host system and the associated - resources of the framework on the current system. - It also encapsulates other runtime parameters that are relevant to the - framework's execution. - - There is a single instance of this class globally in the framework. - - .. note:: - .. versionadded:: 2.13 - - ''' - - def __init__(self, dict_config, sysdescr=None, **options): - self._site_config = config.SiteConfiguration(dict_config) - if sysdescr is not None: - sysname, _, partname = sysdescr.partition(':') - try: - self._system = HostSystem( - self._site_config.systems[sysname], partname) - except KeyError: - raise UnknownSystemError('unknown system: %s' % - sysdescr) from None - else: - self._system = HostSystem(self._autodetect_system()) - - self._resources = HostResources( - self._system.prefix, self._system.stagedir, - self._system.outputdir, self._system.perflogdir) - self._modules_system = ModulesSystem.create( - self._system.modules_system) - self._current_run = 0 - self._non_default_craype = options.get('non_default_craype', False) - - def _autodetect_system(self): - '''Auto-detect system.''' - - # Try to detect directly the cluster name from /etc/xthostname (Cray - # specific) - try: - hostname = os_ext.run_command( - 'cat /etc/xthostname', check=True).stdout - except SpawnedProcessError: - # Try to figure it out with the standard method - hostname = socket.gethostname() - - # Go through the supported systems and try to match the hostname - for system in self._site_config.systems.values(): - for hostname_patt in system.hostnames: - if re.match(hostname_patt, hostname): - return system - - raise SystemAutodetectionError - - def mode(self, name): - try: - return self._site_config.modes[name] - except KeyError: - raise ConfigError('unknown execution mode: %s' % name) from None - def next_run(self): self._current_run += 1 @@ -245,6 +68,10 @@ def next_run(self): def current_run(self): return self._current_run + @property + def site_config(self): + return self._site_config + @property def system(self): '''The current host system. @@ -254,55 +81,85 @@ def system(self): return self._system @property - def resources(self): - '''The framework resources. - - :type: :class:`reframe.core.runtime.HostResources` - ''' - return self._resources + def prefix(self): + return os_ext.expandvars( + self.site_config.get('systems/0/prefix') + ) @property - def modules_system(self): - '''The modules system used by the current host system. + def stagedir(self): + return os_ext.expandvars( + self.site_config.get('systems/0/stagedir') + ) - :type: :class:`reframe.core.modules.ModulesSystem`. - ''' - return self._modules_system + @property + def outputdir(self): + return os_ext.expandvars( + self.site_config.get('systems/0/outputdir') + ) @property - def non_default_craype(self): - '''True if a non-default Cray PE is tested. + def perflogdir(self): + # Find the first filelog handler + handlers = self.site_config.get('logging/0/handlers_perflog') + for i, h in enumerate(handlers): + if h['type'] == 'filelog': + break - This will cause ReFrame to set the ``LD_LIBRARY_PATH`` as follows after - all modules have been loaded: + return os_ext.expandvars( + self.site_config.get(f'logging/0/handlers_perflog/{i}/basedir') + ) - .. code:: shell + @property + def timestamp(self): + timefmt = self.site_config.get('general/0/timestamp_dirs') + return self._timestamp.strftime(timefmt) - export LD_LIBRARY_PATH=$CRAY_LD_LIBRARY_PATH:$LD_LIBRARY_PATH + @property + def output_prefix(self): + '''The output prefix directory of ReFrame.''' + if self.outputdir: + return os.path.join(self.outputdir, self.timestamp) + else: + return os.path.join(self.prefix, 'output', self.timestamp) + @property + def stage_prefix(self): + '''The stage prefix directory of ReFrame.''' + if self.stagedir: + return os.path.join(self.stagedir, self.timestamp) + else: + return os.path.join(self.prefix, 'stage', self.timestamp) - This property is set through the ``--non-default-craype`` command-line - option. + def make_stagedir(self, *dirs, wipeout=True): + return self._makedir(self.stage_prefix, + *self._format_dirs(*dirs), wipeout=wipeout) - :type: :class:`bool` (default: :class:`False`) + def make_outputdir(self, *dirs, wipeout=True): + return self._makedir(self.output_prefix, + *self._format_dirs(*dirs), wipeout=wipeout) + @property + def modules_system(self): + '''The modules system used by the current host system. + + :type: :class:`reframe.core.modules.ModulesSystem`. ''' - return self._non_default_craype + return self._system.modules_system - def show_config(self): - '''Return a textual representation of the current runtime.''' - return str(self._system) + def get_option(self, option): + return self._site_config.get(option) # Global resources for the current host _runtime_context = None -def init_runtime(dict_config, sysname=None, **options): +def init_runtime(site_config): global _runtime_context if _runtime_context is None: - _runtime_context = RuntimeContext(dict_config, sysname, **options) + _runtime_context = RuntimeContext(site_config) def runtime(): @@ -319,18 +176,80 @@ def runtime(): return _runtime_context +def loadenv(*environs): + '''Load environments in the current Python context. + + Returns a tuple containing a snapshot of the environment at entry to this + function and a list of shell commands required to load ``environs``. + ''' + modules_system = runtime().modules_system + env_snapshot = snapshot() + commands = [] + for env in environs: + for m in env.modules: + conflicted = modules_system.load_module(m, force=True) + for c in conflicted: + commands += modules_system.emit_unload_commands(c) + + commands += modules_system.emit_load_commands(m) + + for k, v in env.variables.items(): + os.environ[k] = os_ext.expandvars(v) + commands.append('export %s=%s' % (k, v)) + + return env_snapshot, commands + + +def emit_loadenv_commands(*environs): + env_snapshot, commands = loadenv(*environs) + env_snapshot.restore() + return commands + + +def is_env_loaded(environ): + ''':class:`True` if this environment is loaded, :class:`False` otherwise. + ''' + is_module_loaded = runtime().modules_system.is_module_loaded + return (all(map(is_module_loaded, environ.modules)) and + all(os.environ.get(k, None) == os_ext.expandvars(v) + for k, v in environ.variables.items())) + + +class temp_environment: + '''Context manager to temporarily change the environment.''' + + def __init__(self, modules=[], variables=[]): + self._modules = modules + self._variables = variables + + def __enter__(self): + new_env = Environment('_rfm_temp_env', self._modules, self._variables) + self._environ_save, _ = loadenv(new_env) + return new_env + + def __exit__(self, exc_type, exc_value, traceback): + self._environ_save.restore() + + # The following utilities are useful only for the unit tests class temp_runtime: '''Context manager to temporarily switch to another runtime.''' - def __init__(self, dict_config, sysname=None): + def __init__(self, config_file, sysname=None, options=None): global _runtime_context + + options = options or {} self._runtime_save = _runtime_context - if dict_config is None: + if config_file is None: _runtime_context = None else: - _runtime_context = RuntimeContext(dict_config, sysname) + site_config = config.load_config(config_file) + site_config.select_subconfig(sysname) + for opt, value in options.items(): + site_config.add_sticky_option(opt, value) + + _runtime_context = RuntimeContext(site_config) def __enter__(self): return _runtime_context @@ -340,13 +259,13 @@ def __exit__(self, exc_type, exc_value, traceback): _runtime_context = self._runtime_save -def switch_runtime(dict_config, sysname=None): +def switch_runtime(config_file, sysname=None, options=None): '''Function decorator for temporarily changing the runtime for a function.''' def _runtime_deco(fn): @functools.wraps(fn) def _fn(*args, **kwargs): - with temp_runtime(dict_config, sysname): + with temp_runtime(config_file, sysname, options): ret = fn(*args, **kwargs) return ret diff --git a/reframe/core/schedulers/__init__.py b/reframe/core/schedulers/__init__.py index 47d4e85c9f..d7599d69da 100644 --- a/reframe/core/schedulers/__init__.py +++ b/reframe/core/schedulers/__init__.py @@ -10,8 +10,8 @@ import abc import time -import reframe.core.environments as env import reframe.core.fields as fields +import reframe.core.runtime as runtime import reframe.core.shell as shell import reframe.utility.typecheck as typ from reframe.core.exceptions import JobError, JobNotStartedError @@ -101,7 +101,7 @@ class Job: #: #: .. code:: python #: - #: from reframe.core.launchers.registry import getlauncher + #: from reframe.core.backends import getlauncher #: #: @rfm.run_after('setup') #: def set_launcher(self): @@ -308,7 +308,7 @@ def prepare(self, commands, environs=None, **gen_opts): with shell.generate_script(self.script_filename, **gen_opts) as builder: builder.write_prolog(self.scheduler.emit_preamble(self)) - builder.write(env.emit_load_commands(*environs)) + builder.write(runtime.emit_loadenv_commands(*environs)) for c in commands: builder.write_body(c) diff --git a/reframe/core/schedulers/local.py b/reframe/core/schedulers/local.py index 6d4bbf7730..2f708f756f 100644 --- a/reframe/core/schedulers/local.py +++ b/reframe/core/schedulers/local.py @@ -13,9 +13,9 @@ import reframe.core.schedulers as sched import reframe.utility.os_ext as os_ext +from reframe.core.backends import register_scheduler from reframe.core.exceptions import ReframeError from reframe.core.logging import getlogger -from reframe.core.schedulers.registry import register_scheduler class _TimeoutExpired(ReframeError): diff --git a/reframe/core/schedulers/pbs.py b/reframe/core/schedulers/pbs.py index 74310d8247..11b4374e33 100644 --- a/reframe/core/schedulers/pbs.py +++ b/reframe/core/schedulers/pbs.py @@ -16,12 +16,13 @@ import time from datetime import datetime +import reframe.core.runtime as rt import reframe.core.schedulers as sched import reframe.utility.os_ext as os_ext +from reframe.core.backends import register_scheduler from reframe.core.config import settings from reframe.core.exceptions import SpawnedProcessError, JobError from reframe.core.logging import getlogger -from reframe.core.schedulers.registry import register_scheduler from reframe.utility import seconds_to_hms @@ -30,6 +31,12 @@ PBS_OUTPUT_WRITEBACK_WAIT = 3 +# Minimum amount of time between its submission and its cancellation. If you +# immediately cancel a PBS job after submission, its output files may never +# appear in the output causing the wait() to hang. +PBS_CANCEL_DELAY = 3 + + _run_strict = functools.partial(os_ext.run_command, check=True) @@ -41,6 +48,9 @@ class PbsJobScheduler(sched.JobScheduler): def __init__(self): self._prefix = '#PBS' self._time_finished = None + self._job_submit_timeout = rt.runtime().get_option( + f'schedulers/@{self.registered_name}/job_submit_timeout' + ) # Optional part of the job id refering to the PBS server self._pbs_server = None @@ -111,7 +121,7 @@ def submit(self, job): # Slurm wrappers. cmd = 'qsub -o %s -e %s %s' % (job.stdout, job.stderr, job.script_filename) - completed = _run_strict(cmd, timeout=settings().job_submit_timeout) + completed = _run_strict(cmd, timeout=self._job_submit_timeout) jobid_match = re.search(r'^(?P\S+)', completed.stdout) if not jobid_match: raise JobError('could not retrieve the job id ' @@ -125,7 +135,7 @@ def submit(self, job): self._submit_time = datetime.now() def wait(self, job): - intervals = itertools.cycle(settings().job_poll_intervals) + intervals = itertools.cycle([1, 2, 3]) while not self.finished(job): time.sleep(next(intervals)) @@ -135,8 +145,12 @@ def cancel(self, job): if self._pbs_server: jobid += '.' + self._pbs_server + time_from_submit = (datetime.now() - self._submit_time).total_seconds() + if time_from_submit < PBS_CANCEL_DELAY: + time.sleep(PBS_CANCEL_DELAY - time_from_submit) + getlogger().debug('cancelling job (id=%s)' % jobid) - _run_strict('qdel %s' % jobid, timeout=settings().job_submit_timeout) + _run_strict('qdel %s' % jobid, timeout=self._job_submit_timeout) def finished(self, job): with os_ext.change_dir(job.workdir): diff --git a/reframe/core/schedulers/slurm.py b/reframe/core/schedulers/slurm.py index a604f817f4..279774899b 100644 --- a/reframe/core/schedulers/slurm.py +++ b/reframe/core/schedulers/slurm.py @@ -13,13 +13,13 @@ from datetime import datetime import reframe.core.environments as env +import reframe.core.runtime as rt import reframe.core.schedulers as sched import reframe.utility.os_ext as os_ext -from reframe.core.config import settings +from reframe.core.backends import register_scheduler from reframe.core.exceptions import (SpawnedProcessError, JobBlockedError, JobError) from reframe.core.logging import getlogger -from reframe.core.schedulers.registry import register_scheduler from reframe.utility import seconds_to_hms @@ -104,13 +104,16 @@ def __init__(self): self._update_state_count = 0 self._submit_time = None self._completion_time = None + self._job_submit_timeout = rt.runtime().get_option( + f'schedulers/@{self.registered_name}/job_submit_timeout' + ) def completion_time(self, job): if (self._completion_time or not slurm_state_completed(job.state)): return self._completion_time - with env.temp_environment(variables={'SLURM_TIME_FORMAT': '%s'}): + with rt.temp_environment(variables={'SLURM_TIME_FORMAT': '%s'}): completed = os_ext.run_command( 'sacct -S %s -P -j %s -o jobid,end' % (self._submit_time.strftime('%F'), job.jobid), @@ -188,7 +191,7 @@ def emit_preamble(self, job): def submit(self, job): cmd = 'sbatch %s' % job.script_filename - completed = _run_strict(cmd, timeout=settings().job_submit_timeout) + completed = _run_strict(cmd, timeout=self._job_submit_timeout) jobid_match = re.search(r'Submitted batch job (?P\d+)', completed.stdout) if not jobid_match: @@ -420,7 +423,7 @@ def wait(self, job): return - intervals = itertools.cycle(settings().job_poll_intervals) + intervals = itertools.cycle([1, 2, 3]) self._update_state(job) while not slurm_state_completed(job.state): @@ -438,8 +441,7 @@ def wait(self, job): def cancel(self, job): getlogger().debug('cancelling job (id=%s)' % job.jobid) - _run_strict('scancel %s' % job.jobid, - timeout=settings().job_submit_timeout) + _run_strict('scancel %s' % job.jobid, timeout=self._job_submit_timeout) self._is_cancelling = True def finished(self, job): @@ -555,6 +557,7 @@ def __init__(self, node_descr): 'ActiveFeatures', node_descr, sep=',') or set() self._states = self._extract_attribute( 'State', node_descr, sep='+') or set() + self._descr = node_descr def __eq__(self, other): if not isinstance(other, type(self)): @@ -588,6 +591,10 @@ def partitions(self): def states(self): return self._states + @property + def descr(self): + return self._descr + def _extract_attribute(self, attr_name, node_descr, sep=None): attr_match = re.search(r'%s=(\S+)' % attr_name, node_descr) if attr_match: diff --git a/reframe/core/schedulers/torque.py b/reframe/core/schedulers/torque.py index 0cf46cee33..2980333cac 100644 --- a/reframe/core/schedulers/torque.py +++ b/reframe/core/schedulers/torque.py @@ -13,11 +13,10 @@ from datetime import datetime import reframe.utility.os_ext as os_ext -from reframe.core.config import settings +from reframe.core.backends import register_scheduler from reframe.core.exceptions import JobError from reframe.core.logging import getlogger from reframe.core.schedulers.pbs import PbsJobScheduler, _run_strict -from reframe.core.schedulers.registry import register_scheduler JOB_STATES = { diff --git a/reframe/core/settings.py b/reframe/core/settings.py new file mode 100644 index 0000000000..e7894616ee --- /dev/null +++ b/reframe/core/settings.py @@ -0,0 +1,71 @@ +# Copyright 2016-2020 Swiss National Supercomputing Centre (CSCS/ETH Zurich) +# ReFrame Project Developers. See the top-level LICENSE file for details. +# +# SPDX-License-Identifier: BSD-3-Clause + +# +# Generic fallback configuration +# + +site_configuration = { + 'systems': [ + { + 'name': 'generic', + 'descr': 'Generic example system', + 'hostnames': ['.*'], + 'partitions': [ + { + 'name': 'default', + 'scheduler': 'local', + 'launcher': 'local', + 'environs': ['builtin'] + } + ] + }, + ], + 'environments': [ + { + 'name': 'builtin', + 'cc': 'cc', + 'cxx': '', + 'ftn': '' + }, + ], + 'logging': [ + { + 'level': 'debug', + 'handlers': [ + { + 'type': 'stream', + 'name': 'stdout', + 'level': 'info', + 'format': '%(message)s' + }, + { + 'type': 'file', + 'name': 'reframe.log', + 'level': 'debug', + 'format': '[%(asctime)s] %(levelname)s: %(check_info)s: %(message)s', # noqa: E501 + 'append': False + } + ], + 'handlers_perflog': [ + { + 'type': 'filelog', + 'prefix': '%(check_system)s/%(check_partition)s', + 'level': 'info', + 'format': ( + '%(check_job_completion_time)s|reframe %(version)s|' + '%(check_info)s|jobid=%(check_jobid)s|' + '%(check_perf_var)s=%(check_perf_value)s|' + 'ref=%(check_perf_ref)s ' + '(l=%(check_perf_lower_thres)s, ' + 'u=%(check_perf_upper_thres)s)|' + '%(check_perf_unit)s' + ), + 'append': True + } + ] + } + ], +} diff --git a/reframe/core/systems.py b/reframe/core/systems.py index 28f430a459..5c7930ee29 100644 --- a/reframe/core/systems.py +++ b/reframe/core/systems.py @@ -3,49 +3,30 @@ # # SPDX-License-Identifier: BSD-3-Clause +import json import re -import reframe.core.debug as debug -import reframe.core.fields as fields import reframe.utility as utility -import reframe.utility.typecheck as typ -from reframe.core.environments import Environment +from reframe.core.backends import (getlauncher, getscheduler) +from reframe.core.modules import ModulesSystem +from reframe.core.environments import (Environment, ProgEnvironment) -class SystemPartition: - '''A representation of a system partition inside ReFrame. - - This class is immutable. - ''' - - _name = fields.TypedField('_name', typ.Str[r'(\w|-)+']) - _descr = fields.TypedField('_descr', str) - _access = fields.TypedField('_access', typ.List[str]) - _environs = fields.TypedField('_environs', typ.List[Environment]) - _resources = fields.TypedField('_resources', typ.Dict[str, typ.List[str]]) - _local_env = fields.TypedField('_local_env', Environment, type(None)) - _container_environs = fields.TypedField('_container_environs', - typ.Dict[str, Environment]) - - # maximum concurrent jobs - _max_jobs = fields.TypedField('_max_jobs', int) - - def __init__(self, name, descr=None, scheduler=None, launcher=None, - access=[], environs=[], resources={}, local_env=None, - max_jobs=1): - self._name = name - self._descr = descr or name +class _SystemPartition: + def __init__(self, parent, name, scheduler, launcher, + descr, access, container_environs, resources, + local_env, environs, max_jobs): + self._parent_system = parent + self._name = name self._scheduler = scheduler - self._launcher = launcher - self._access = list(access) - self._environs = list(environs) - self._resources = dict(resources) - self._max_jobs = max_jobs + self._launcher = launcher + self._descr = descr + self._access = access + self._container_environs = container_environs self._local_env = local_env - self._container_environs = {} - - # Parent system - self._system = None + self._environs = environs + self._max_jobs = max_jobs + self._resources = {r['name']: r['options'] for r in resources} @property def access(self): @@ -73,10 +54,7 @@ def fullname(self): :type: `str` ''' - if self._system is None: - return self._name - else: - return '%s:%s' % (self._system.name, self._name) + return f'{self._parent_system}:{self._name}' @property def local_env(self): @@ -102,7 +80,7 @@ def resources(self): def scheduler(self): '''The type of the backend scheduler of this partition. - :returns: a subclass of :class:`reframe.core.schedulers.Job`. + :returns: a subclass of :class:`reframe.core.schedulers.JobScheduler`. .. note:: .. versionchanged:: 2.8 @@ -123,9 +101,6 @@ def launcher(self): ''' return self._launcher - def add_container_env(self, env_name, environ): - self._container_environs[env_name] = environ - # Instantiate managed resource `name` with `value`. def get_resource(self, name, **values): ret = [] @@ -138,7 +113,7 @@ def get_resource(self, name, **values): return ret def environment(self, name): - for e in self._environs: + for e in self.environs: if e.name == name: return e @@ -156,56 +131,136 @@ def __eq__(self, other): self._resources == other._resources and self._local_env == other._local_env) - def __str__(self): - local_env = re.sub('(?m)^', 6*' ', ' - ' + self._local_env.details()) - lines = [ - '%s [%s]:' % (self._name, self._descr), - ' fullname: ' + self.fullname, - ' scheduler: ' + self._scheduler.registered_name, - ' launcher: ' + self._launcher.registered_name, - ' access: ' + ' '.join(self._access), - ' local_env:\n' + local_env, - ' environs: ' + ', '.join(str(e) for e in self._environs) - ] - return '\n'.join(lines) + def json(self): + return { + 'name': self._name, + 'descr': self._descr, + 'scheduler': self._scheduler.registered_name, + 'launcher': self._launcher.registered_name, + 'access': self._access, + 'container_platforms': [ + { + 'type': ctype, + 'modules': [m.name for m in cpenv.modules], + 'variables': [[n, v] for n, v in cpenv.variables.items()] + } + for ctype, cpenv in self._container_environs.items() + ], + 'modules': [m.name for m in self._local_env.modules], + 'variables': [[n, v] + for n, v in self._local_env.variables.items()], + 'environs': [e.name for e in self._environs], + 'max_jobs': self._max_jobs, + 'resources': [ + { + 'name': name, + 'options': options + } + for name, options in self._resources.items() + ] + } - def __repr__(self): - return debug.repr(self) + def __str__(self): + return json.dumps(self.json(), indent=2) class System: '''A representation of a system inside ReFrame.''' - _name = fields.TypedField('_name', typ.Str[r'(\w|-)+']) - _descr = fields.TypedField('_descr', str) - _hostnames = fields.TypedField('_hostnames', typ.List[str]) - _partitions = fields.TypedField('_partitions', typ.List[SystemPartition]) - _modules_system = fields.TypedField('_modules_system', - typ.Str[r'(\w|-)+'], type(None)) - _preload_env = fields.TypedField('_preload_env', Environment, type(None)) - _prefix = fields.TypedField('_prefix', str) - _stagedir = fields.TypedField('_stagedir', str, type(None)) - _outputdir = fields.TypedField('_outputdir', str, type(None)) - _perflogdir = fields.TypedField('_perflogdir', str, type(None)) - _resourcesdir = fields.TypedField('_resourcesdir', str) - - def __init__(self, name, descr=None, hostnames=[], partitions=[], - preload_env=None, prefix='.', stagedir=None, outputdir=None, - perflogdir=None, resourcesdir='.', modules_system=None): - self._name = name - self._descr = descr or name - self._hostnames = list(hostnames) - self._partitions = list(partitions) - self._modules_system = modules_system + + def __init__(self, name, descr, hostnames, modules_system, + preload_env, prefix, outputdir, + resourcesdir, stagedir, partitions): + self._name = name + self._descr = descr + self._hostnames = hostnames + self._modules_system = ModulesSystem.create(modules_system) self._preload_env = preload_env self._prefix = prefix - self._stagedir = stagedir self._outputdir = outputdir - self._perflogdir = perflogdir self._resourcesdir = resourcesdir - - # Set parent system for the given partitions - for p in partitions: - p._system = self + self._stagedir = stagedir + self._partitions = partitions + + @classmethod + def create(cls, site_config): + # Create the whole system hierarchy from bottom up + sysname = site_config.get('systems/0/name') + partitions = [] + config_save = site_config.subconfig_system + for p in site_config.get('systems/0/partitions'): + site_config.select_subconfig(f'{sysname}:{p["name"]}') + partid = f"systems/0/partitions/@{p['name']}" + part_name = site_config.get(f'{partid}/name') + part_sched = getscheduler(site_config.get(f'{partid}/scheduler')) + part_launcher = getlauncher(site_config.get(f'{partid}/launcher')) + part_container_environs = {} + for i, p in enumerate( + site_config.get(f'{partid}/container_platforms') + ): + ctype = p['type'] + part_container_environs[ctype] = Environment( + name=f'__rfm_env_{ctype}', + modules=site_config.get( + f'{partid}/container_platforms/{i}/modules' + ), + variables=site_config.get( + f'{partid}/container_platforms/{i}/variables' + ) + ) + + part_environs = [ + ProgEnvironment( + name=e, + modules=site_config.get(f'environments/@{e}/modules'), + variables=site_config.get(f'environments/@{e}/variables'), + cc=site_config.get(f'environments/@{e}/cc'), + cxx=site_config.get(f'environments/@{e}/cxx'), + ftn=site_config.get(f'environments/@{e}/ftn'), + cppflags=site_config.get(f'environments/@{e}/cppflags'), + cflags=site_config.get(f'environments/@{e}/cflags'), + cxxflags=site_config.get(f'environments/@{e}/cxxflags'), + fflags=site_config.get(f'environments/@{e}/fflags'), + ldflags=site_config.get(f'environments/@{e}/ldflags') + ) for e in site_config.get(f'{partid}/environs') + ] + partitions.append( + _SystemPartition( + parent=site_config.get('systems/0/name'), + name=part_name, + scheduler=part_sched, + launcher=part_launcher, + descr=site_config.get(f'{partid}/descr'), + access=site_config.get(f'{partid}/access'), + resources=site_config.get(f'{partid}/resources'), + environs=part_environs, + container_environs=part_container_environs, + local_env=Environment( + name=f'__rfm_env_{part_name}', + modules=site_config.get(f'{partid}/modules'), + variables=site_config.get(f'{partid}/variables') + ), + max_jobs=site_config.get(f'{partid}/max_jobs') + ) + ) + + # Restore configuration + site_config.select_subconfig(config_save) + return System( + name=sysname, + descr=site_config.get('systems/0/descr'), + hostnames=site_config.get('systems/0/hostnames'), + modules_system=site_config.get('systems/0/modules_system'), + preload_env=Environment( + name=f'__rfm_env_{sysname}', + modules=site_config.get('systems/0/modules'), + variables=site_config.get('systems/0/variables') + ), + prefix=site_config.get('systems/0/prefix'), + outputdir=site_config.get('systems/0/outputdir'), + resourcesdir=site_config.get('systems/0/resourcesdir'), + stagedir=site_config.get('systems/0/stagedir'), + partitions=partitions + ) @property def name(self): @@ -251,11 +306,6 @@ def outputdir(self): '''The ReFrame output directory prefix associated with this system.''' return self._outputdir - @property - def perflogdir(self): - '''The ReFrame log directory prefix associated with this system.''' - return self._perflogdir - @property def resourcesdir(self): '''Global resources directory for this system. @@ -274,10 +324,6 @@ def partitions(self): '''All the system partitions associated with this system.''' return utility.SequenceView(self._partitions) - def add_partition(self, partition): - partition._system = self - self._partitions.append(partition) - def __eq__(self, other): if not isinstance(other, type(self)): return NotImplemented @@ -286,5 +332,35 @@ def __eq__(self, other): self._hostnames == other._hostnames and self._partitions == other._partitions) + def json(self): + return { + 'name': self._name, + 'descr': self._descr, + 'hostnames': self._hostnames, + 'modules_system': self._modules_system.name, + 'modules': [m.name for m in self._preload_env.modules], + 'variables': [ + [name, value] + for name, value in self._preload_env.variables.items() + ], + 'prefix': self._prefix, + 'outputdir': self._outputdir, + 'stagedir': self._stagedir, + 'resourcesdir': self._resourcesdir, + 'partitions': [p.json() for p in self._partitions] + } + + def __str__(self): + return json.dumps(self.json(), indent=2) + def __repr__(self): - return debug.repr(self) + return ( + f'{type(self).__name__}( ' + f'name={self._name!r}, descr={self._descr!r}, ' + f'hostnames={self._hostnames!r}, ' + f'modules_system={self.modules_system.name!r}, ' + f'preload_env={self._preload_env!r}, prefix={self._prefix!r}, ' + f'outputdir={self._outputdir!r}, ' + f'resourcesdir={self._resourcesdir!r}, ' + f'stagedir={self._stagedir!r}, partitions={self._partitions!r})' + ) diff --git a/reframe/frontend/argparse.py b/reframe/frontend/argparse.py index cfd1841c98..5cf672fa47 100644 --- a/reframe/frontend/argparse.py +++ b/reframe/frontend/argparse.py @@ -4,9 +4,7 @@ # SPDX-License-Identifier: BSD-3-Clause import argparse - -from reframe.core.fields import ForwardField - +import os # # Notes on the ArgumentParser design @@ -20,7 +18,7 @@ # For this reason, we base our design on composition by implementing wrappers # of both the argument group and the argument parser. These wrappers provide # the same public interface as their `argparse` counterparts (currently we only -# implement the part of the interface that matters for Reframe), delegating the +# implement the part of the interface that matters for ReFrame), delegating the # parsing work to them. For these "shadow" data structures for argument groups # and the parser, we follow a similar design as in the `argparse` module: both # the argument group and the parser inherit from a base class implementing the @@ -28,49 +26,177 @@ # # A final trick we had to do in order to avoid repeating all the public fields # of the internal argument holders (`argparse`'s argument group or argument -# parser) was to programmaticallly export them by creating special descriptor -# fields that forward the set/get actions to the internal argument holder. +# parser) was to programmaticallly export them by implementing the +# `__getattr__()` method, such as to delegate any lookup of unknown public +# attributes to the underlying `argparse.ArgumentParser`. # +# Finally, the functionality of the ArgumentParser is extended to support +# associations of command-line arguments with environment variables and/or +# configuration parameters. Additionally, we allow to define pseudo-arguments +# that essentially associate environment variables with configuration +# arguments, without having to define a corresponding command line option. + + +def _convert_to_bool(s): + if s.lower() in ('true', 'yes', 'y'): + return True + + if s.lower() in ('false', 'no', 'n'): + return False + + raise ValueError + + +class _Namespace: + def __init__(self, namespace, option_map): + self.__namespace = namespace + self.__option_map = option_map + + @property + def cmd_options(self): + '''Options filled in by command-line''' + return self.__namespace + + def __getattr__(self, name): + if name.startswith('_'): + raise AttributeError( + f"'{type(self).__name__}' object has no attribute '{name}'" + ) + + try: + ret = getattr(self.__namespace, name) + except AttributeError: + if name not in self.__option_map: + # Option not defined at all + raise + + # Option is not associated with a command-line argument + ret = None + + if name not in self.__option_map: + return ret + + envvar, _, action = self.__option_map[name] + if ret is None and envvar is not None: + # Try the environment variable + envvar, *delim = envvar.split(maxsplit=2) + delim = delim[0] if delim else ',' + ret = os.getenv(envvar) + if ret is not None: + if action.startswith('append'): + # The option should be interpreted as comma separated list + ret = ret.split(delim) + elif action in ('store_true', 'store_false'): + try: + ret = _convert_to_bool(ret) + except ValueError: + raise ValueError( + f'environment variable {envvar!r} not a boolean' + ) from None + + return ret + + def update_config(self, site_config): + '''Update the site configuration with the options represented by this + namespace''' + errors = [] + for option, spec in self.__option_map.items(): + _, confvar, action = spec + if action == 'version' or confvar is None: + continue + + try: + value = getattr(self, option) + except ValueError as e: + errors.append(e) + continue + + if value is not None: + site_config.add_sticky_option(confvar, value) + + return errors + + def __repr__(self): + return (f'{type(self).__name__}({self.__namespace!r}, ' + '{self.__option_map})') class _ArgumentHolder: - def __init__(self, holder): + def __init__(self, holder, shared_options=None): self._holder = holder self._defaults = argparse.Namespace() - # Create forward descriptors to all public members of _holder - for m in self._holder.__dict__.keys(): - if m[0] != '_': - setattr(type(self), m, ForwardField(self._holder, m)) + # Map command-line options to environment variables and configuration + # options. Values are tuples of the form (envvar, configvar) + self._option_map = shared_options if shared_options is not None else {} - def _attr_from_flag(self, *flags): - if not flags: - raise ValueError('could not infer a dest name: no flags defined') + def __getattr__(self, name): + # Delegate all unknown public attribute requests to the underlying + # holder + if name.startswith('_'): + raise AttributeError( + f"'{type(self).__name__}' object has no attribute '{name}'" + ) + + return getattr(self._holder, name) - return flags[-1].lstrip('-').replace('-', '_') + def add_argument(self, *flags, **kwargs): + try: + opt_name = kwargs['dest'] + except KeyError: + # Try to figure out the dest name as the original ArgumentParser + opt_name = None + for f in flags: + # The last long option is taken into account as the option + # name + if f.startswith('--'): + opt_name = f[2:].replace('-', '_') + + if flags and opt_name is None: + # The first short option is taken into account as the + # option name + if flags[0].startswith('-'): + opt_name = flags[0][1:].replace('-', '_') + + if flags and opt_name is None: + # A positional argument + opt_name = flags[-1] + + if opt_name is None: + raise ValueError('could not infer a dest name: no flags defined') - def _extract_default(self, *flags, **kwargs): - attr = kwargs.get('dest', self._attr_from_flag(*flags)) + self._option_map[opt_name] = ( + kwargs.get('envvar', None), + kwargs.get('configvar', None), + kwargs.get('action', 'store') + ) + # Remove envvar and configvar keyword arguments and force dest + # argument, even if we guessed it, in order to guard against changes + # in ArgumentParser's implementation + kwargs.pop('envvar', None) + kwargs.pop('configvar', None) + kwargs['dest'] = opt_name + + # Convert 'store_true' and 'store_false' actions to their + # 'store_const' equivalents, because they otherwise imply imply a + # default action = kwargs.get('action', None) if action == 'store_true' or action == 'store_false': - # These actions imply a default; we will convert them to their - # 'const' action equivalent and add an explicit default value kwargs['action'] = 'store_const' - kwargs['const'] = True if action == 'store_true' else False - kwargs['default'] = False if action == 'store_true' else True + kwargs['const'] = True if action == 'store_true' else False + kwargs['const'] = False if action == 'store_false' else True + # Remove defaults try: - self._defaults.__dict__[attr] = kwargs['default'] + self._defaults.__dict__[opt_name] = kwargs['default'] del kwargs['default'] except KeyError: - self._defaults.__dict__[attr] = None - finally: - return kwargs + self._defaults.__dict__[opt_name] = None - def add_argument(self, *flags, **kwargs): - return self._holder.add_argument( - *flags, **self._extract_default(*flags, **kwargs) - ) + if not flags: + return None + + return self._holder.add_argument(*flags, **kwargs) class _ArgumentGroup(_ArgumentHolder): @@ -91,7 +217,9 @@ def __init__(self, **kwargs): def add_argument_group(self, *args, **kwargs): group = _ArgumentGroup( - self._holder.add_argument_group(*args, **kwargs)) + self._holder.add_argument_group(*args, **kwargs), + self._option_map + ) self._groups.append(group) return group @@ -110,9 +238,6 @@ def _update_defaults(self): for g in self._groups: self._defaults.__dict__.update(g._defaults.__dict__) - def print_help(self): - self._holder.print_help() - def parse_args(self, args=None, namespace=None): '''Convert argument strings to objects and return them as attributes of a namespace. @@ -134,6 +259,11 @@ def parse_args(self, args=None, namespace=None): # do this in options with an 'append' action. options = self._holder.parse_args(args, None) + # Check if namespace refers to our namespace and take the cmd options + # namespace suitable for ArgumentParser + if isinstance(namespace, _Namespace): + namespace = namespace.cmd_options + # Update parser's defaults with groups' defaults self._update_defaults() for attr, val in options.__dict__.items(): @@ -142,12 +272,14 @@ def parse_args(self, args=None, namespace=None): attr, [namespace, self._defaults] ) - return options + return _Namespace(options, self._option_map) def format_options(namespace): '''Format parsed arguments in ``namespace``.''' ret = 'Command-line configuration:\n' - ret += '\n'.join([' %s=%s' % (attr, val) - for attr, val in sorted(namespace.__dict__.items())]) + ret += '\n'.join( + [' %s=%s' % (attr, val) + for attr, val in sorted(namespace.cmd_options.__dict__.items())] + ) return ret diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index fd60cdc47e..acff872003 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -4,12 +4,12 @@ # SPDX-License-Identifier: BSD-3-Clause import inspect +import json import os import re import socket import sys import traceback -import warnings import reframe import reframe.core.config as config @@ -21,10 +21,10 @@ import reframe.frontend.dependency as dependency import reframe.utility.os_ext as os_ext from reframe.core.exceptions import ( - ConfigError, EnvironError, ReframeDeprecationWarning, ReframeError, - ReframeFatalError, ReframeForceExitError, SystemAutodetectionError + EnvironError, ConfigError, ReframeError, + ReframeDeprecationWarning, ReframeFatalError, + format_exception, SystemAutodetectionError ) -from reframe.core.exceptions import format_exception from reframe.frontend.executors import Runner, generate_testcases from reframe.frontend.executors.policies import (SerialExecutionPolicy, AsynchronousExecutionPolicy) @@ -33,33 +33,32 @@ def format_check(check, detailed): - lines = [' * %s (found in %s)' % (check.name, + lines = [' - %s (found in %s)' % (check.name, inspect.getfile(type(check)))] flex = 'flexible' if check.num_tasks <= 0 else 'standard' if detailed: lines += [ - f" - description: {check.descr}", - f" - systems: {', '.join(check.valid_systems)}", - f" - environments: {', '.join(check.valid_prog_environs)}", - f" - modules: {', '.join(check.modules)}", - f" - task allocation: {flex}", - f" - dependencies: " + f" description: {check.descr}", + f" systems: {', '.join(check.valid_systems)}", + f" environments: {', '.join(check.valid_prog_environs)}", + f" modules: {', '.join(check.modules)}", + f" task allocation: {flex}", + f" dependencies: " f"{', '.join([d[0] for d in check.user_deps()])}", - f" - tags: {', '.join(check.tags)}", - f" - maintainers: {', '.join(check.maintainers)}" + f" tags: {', '.join(check.tags)}", + f" maintainers: {', '.join(check.maintainers)}" ] return '\n'.join(lines) def list_checks(checks, printer, detailed=False): - printer.info('List of matched checks') - printer.info('======================') + printer.info('[List of matched checks]') for c in checks: printer.info(format_check(c, detailed)) - printer.info('Found %d check(s).' % len(checks)) + printer.info('\nFound %d check(s).' % len(checks)) def main(): @@ -81,51 +80,75 @@ def main(): # Output directory options output_options.add_argument( '--prefix', action='store', metavar='DIR', - help='Set output directory prefix to DIR') + help='Set output directory prefix to DIR', + envvar='RFM_PREFIX', configvar='systems/prefix' + ) output_options.add_argument( '-o', '--output', action='store', metavar='DIR', - help='Set output directory to DIR') + help='Set output directory to DIR', + envvar='RFM_OUTPUT_DIR', configvar='systems/outputdir' + ) output_options.add_argument( '-s', '--stage', action='store', metavar='DIR', - help='Set stage directory to DIR') + help='Set stage directory to DIR', + envvar='RFM_STAGE_DIR', configvar='systems/stagedir' + ) output_options.add_argument( '--perflogdir', action='store', metavar='DIR', - help='Set directory prefix for the performance logs ' - '(default: ${prefix}/perflogs, ' - 'relevant only if the filelog backend is used)') + help=('Set directory prefix for the performance logs ' + '(default: ${prefix}/perflogs, ' + 'relevant only if the filelog backend is used)'), + envvar='RFM_PERFLOG_DIR', + configvar='logging/handlers_perflog/filelog_basedir' + ) output_options.add_argument( '--keep-stage-files', action='store_true', - help='Keep stage directory even if check is successful') + help='Keep stage directory even if check is successful', + envvar='RFM_KEEP_STAGE_FILES', configvar='general/keep_stage_files' + ) output_options.add_argument( '--save-log-files', action='store_true', default=False, - help='Copy the log file from the work dir to the output dir at the ' - 'end of the program') + help=('Copy the log file from the current directory to the ' + 'output directory when ReFrame ends'), + envvar='RFM_SAVE_LOG_FILES', configvar='general/save_log_files' + ) # Check discovery options locate_options.add_argument( - '-c', '--checkpath', action='store', metavar='DIR|FILE', - help="Search for checks in DIR or FILE; multiple paths can be " - "separated with `:'") + '-c', '--checkpath', action='append', metavar='DIR|FILE', + help="Add DIR or FILE to the check search path", + envvar='RFM_CHECK_SEARCH_PATH :', configvar='general/check_search_path' + ) locate_options.add_argument( '-R', '--recursive', action='store_true', - help='Load checks recursively') + help='Load checks recursively', + envvar='RFM_CHECK_SEARCH_RECURSIVE', + configvar='general/check_search_recursive' + ) locate_options.add_argument( '--ignore-check-conflicts', action='store_true', - help='Skip checks with conflicting names') + help='Skip checks with conflicting names', + envvar='RFM_IGNORE_CHECK_CONFLICTS', + configvar='general/ignore_check_conflicts' + ) # Select options select_options.add_argument( '-t', '--tag', action='append', dest='tags', default=[], - help='Select checks matching TAG') + help='Select checks matching TAG' + ) select_options.add_argument( '-n', '--name', action='append', dest='names', default=[], - metavar='NAME', help='Select checks with NAME') + metavar='NAME', help='Select checks with NAME' + ) select_options.add_argument( '-x', '--exclude', action='append', dest='exclude_names', - metavar='NAME', default=[], help='Exclude checks with NAME') + metavar='NAME', default=[], help='Exclude checks with NAME' + ) select_options.add_argument( '-p', '--prgenv', action='append', default=[r'.*'], - help='Select tests for PRGENV programming environment only') + help='Select tests for PRGENV programming environment only' + ) select_options.add_argument( '--gpu-only', action='store_true', help='Select only GPU tests') @@ -205,67 +228,90 @@ def main(): env_options.add_argument( '-M', '--map-module', action='append', metavar='MAPPING', dest='module_mappings', default=[], - help='Apply a single module mapping') + help='Apply a single module mapping', + envvar='RFM_MODULE_MAPPINGS ,', configvar='general/module_mappings' + ) env_options.add_argument( '-m', '--module', action='append', default=[], metavar='MOD', dest='user_modules', - help='Load module MOD before running the regression suite') + help='Load module MOD before running the regression suite', + envvar='RFM_USER_MODULES', configvar='general/user_modules' + ) env_options.add_argument( '--module-mappings', action='store', metavar='FILE', dest='module_map_file', - help='Apply module mappings defined in FILE') + help='Apply module mappings defined in FILE', + envvar='RFM_MODULE_MAP_FILE', configvar='general/module_map_file' + ) env_options.add_argument( '-u', '--unload-module', action='append', metavar='MOD', dest='unload_modules', default=[], - help='Unload module MOD before running the regression suite') + help='Unload module MOD before running the regression suite', + envvar='RFM_UNLOAD_MODULES', configvar='general/unload_modules' + ) env_options.add_argument( '--purge-env', action='store_true', dest='purge_env', default=False, - help='Purge environment before running the regression suite') + help='Purge environment before running the regression suite', + envvar='RFM_PURGE_ENVIRONMENT', configvar='general/purge_environment' + ) + env_options.add_argument( + '--non-default-craype', action='store_true', + help='Test a non-default Cray PE', + envvar='RFM_NON_DEFAULT_CRAYPE', configvar='general/non_default_craype' + ) # Miscellaneous options misc_options.add_argument( - '-C', '--config-file', action='store', dest='config_file', - metavar='FILE', default=os.path.join(reframe.INSTALL_PREFIX, - 'reframe/settings.py'), - help='Specify a custom config-file for the machine. ' - '(default: %s' % os.path.join(reframe.INSTALL_PREFIX, - 'reframe/settings.py')) - misc_options.add_argument( - '--nocolor', action='store_false', dest='colorize', default=True, - help='Disable coloring of output') - misc_options.add_argument( - '--failure-stats', action='store_true', - help='Print failure statistics') - misc_options.add_argument('--performance-report', action='store_true', - help='Print the performance report') + '-C', '--config-file', action='store', + dest='config_file', metavar='FILE', + help='ReFrame configuration file to use', + envvar='RFM_CONFIG_FILE' + ) misc_options.add_argument( - '--no-deprecation-warnings', action='store_true', - help='Suppress deprecation warnings from the framework') - - # FIXME: This should move to env_options as soon as - # https://github.com/eth-cscs/reframe/pull/946 is merged + '--nocolor', action='store_false', dest='colorize', + help='Disable coloring of output', + envvar='RFM_COLORIZE', configvar='general/colorize' + ) misc_options.add_argument( - '--non-default-craype', action='store_true', default=False, - help='Test a non-default Cray PE') + '--failure-stats', action='store_true', help='Print failure statistics' + ) misc_options.add_argument( - '--show-config', action='store_true', - help='Print configuration of the current system and exit') + '--performance-report', action='store_true', + help='Print a report for performance tests run' + ) misc_options.add_argument( - '--show-config-env', action='store', metavar='ENV', - help='Print configuration of environment ENV and exit') + '--show-config-param', action='store', nargs='?', const='all', + metavar='PARAM', + help=( + 'Print how parameter PARAM is configured ' + 'for the current system and exit' + ) + ) misc_options.add_argument( - '--system', action='store', - help='Load SYSTEM configuration explicitly') + '--system', action='store', help='Load configuration for SYSTEM', + envvar='RFM_SYSTEM' + ) misc_options.add_argument( - '--timestamp', action='store', nargs='?', - const='%FT%T', metavar='TIMEFMT', - help='Append a timestamp component to the regression directories' - '(default format "%%FT%%T")' + '--timestamp', action='store', nargs='?', const='', metavar='TIMEFMT', + help=('Append a timestamp component to the various ' + 'ReFrame directories (default format: "%%FT%%T")'), + envvar='RFM_TIMESTAMP_DIRS', configvar='general/timestamp_dirs' ) misc_options.add_argument('-V', '--version', action='version', version=os_ext.reframe_version()) - misc_options.add_argument('-v', '--verbose', action='count', default=0, - help='Increase verbosity level of output') + misc_options.add_argument( + '-v', '--verbose', action='count', + help='Increase verbosity level of output', + envvar='RFM_VERBOSE', configvar='general/verbose' + ) + + # Options not associated with command-line arguments + argparser.add_argument( + dest='graylog_server', + envvar='RFM_GRAYLOG_SERVER', + configvar='logging/handlers_perflog/graylog_address', + help='Graylog server address' + ) if len(sys.argv) == 1: argparser.print_help() @@ -274,77 +320,61 @@ def main(): # Parse command line options = argparser.parse_args() - # Load configuration - try: - settings = config.load_settings_from_file(options.config_file) - except (OSError, ReframeError) as e: - sys.stderr.write( - '%s: could not load settings: %s\n' % (sys.argv[0], e)) - sys.exit(1) + # First configure logging with our generic configuration so as to be able + # to print pretty messages; logging will be reconfigured by user's + # configuration later + site_config = config.load_config( + os.path.join(reframe.INSTALL_PREFIX, 'reframe/core/settings.py') + ) + site_config.select_subconfig('generic') + options.update_config(site_config) + logging.configure_logging(site_config) + logging.getlogger().colorize = site_config.get('general/0/colorize') + printer = PrettyPrinter() + printer.colorize = site_config.get('general/0/colorize') + if options.verbose: + printer.inc_verbosity(options.verbose) - # Configure logging + # Now configure ReFrame according to the user configuration file try: - logging.configure_logging(settings.logging_config), - except (OSError, ConfigError) as e: - sys.stderr.write('could not configure logging: %s\n' % e) - sys.exit(1) + try: + site_config = config.load_config(options.config_file) + except ReframeDeprecationWarning as e: + printer.warning(e) + converted = config.convert_old_config(options.config_file) + printer.warning( + f"configuration file has been converted " + f"to the new syntax here: '{converted}'" + ) + site_config = config.load_config(converted) - # Set colors in logger - logging.getlogger().colorize = options.colorize + site_config.validate() + site_config.select_subconfig(options.system) + for err in options.update_config(site_config): + printer.warning(str(err)) - # Setup printer - printer = PrettyPrinter() - printer.colorize = options.colorize - if options.verbose: - printer.inc_verbosity(options.verbose) + logging.configure_logging(site_config) + except (OSError, ConfigError) as e: + printer.error(f'failed to load configuration: {e}') + sys.exit(1) + logging.getlogger().colorize = site_config.get('general/0/colorize') + printer.colorize = site_config.get('general/0/colorize') try: - runtime.init_runtime(settings.site_configuration, options.system, - non_default_craype=options.non_default_craype) - except SystemAutodetectionError: - printer.warning( - 'could not find a configuration entry for the current system; ' - 'falling back to a generic system configuration; ' - 'please check the online documentation on how to configure ' - 'ReFrame for your system.' - ) - settings.site_configuration['systems'] = { - 'generic': { - 'descr': 'Generic fallback system configuration', - 'hostnames': ['localhost'], - 'partitions': { - 'login': { - 'scheduler': 'local', - 'environs': ['builtin-gcc'], - 'descr': 'Login nodes' - } - } - } - } - settings.site_configuration['environments'] = { - '*': { - 'builtin-gcc': { - 'type': 'ProgEnvironment', - 'cc': 'gcc', - 'cxx': 'g++', - 'ftn': 'gfortran', - } - } - } - runtime.init_runtime(settings.site_configuration, 'generic', - non_default_craype=options.non_default_craype) - except Exception as e: - printer.error('configuration error: %s' % e) - printer.verbose(''.join(traceback.format_exception(*sys.exc_info()))) + runtime.init_runtime(site_config) + except ConfigError as e: + printer.error(f'failed to initialize runtime: {e}') sys.exit(1) rt = runtime.runtime() try: - if options.module_map_file: - rt.modules_system.load_mapping_from_file(options.module_map_file) + if site_config.get('general/0/module_map_file'): + rt.modules_system.load_mapping_from_file( + site_config.get('general/0/module_map_file') + ) - if options.module_mappings: - for m in options.module_mappings: + if site_config.get('general/0/module_mappings'): + for m in site_config.get('general/0/module_mappings'): rt.modules_system.load_mapping(m) except (ConfigError, OSError) as e: @@ -353,124 +383,65 @@ def main(): if options.mode: try: - mode_args = rt.mode(options.mode) + mode_args = rt.get_option(f'modes/@{options.mode}/options') # Parse the mode's options and reparse the command-line options = argparser.parse_args(mode_args) - options = argparser.parse_args(namespace=options) + options = argparser.parse_args(namespace=options.cmd_options) + options.update_config(rt.site_config) except ConfigError as e: printer.error('could not obtain execution mode: %s' % e) sys.exit(1) - # Adjust system directories - if options.prefix: - # if prefix is set, reset all other directories - rt.resources.prefix = os_ext.expandvars(options.prefix) - rt.resources.outputdir = None - rt.resources.stagedir = None - - if options.output: - rt.resources.outputdir = os_ext.expandvars(options.output) - - if options.stage: - rt.resources.stagedir = os_ext.expandvars(options.stage) - - if (os_ext.samefile(rt.resources.stage_prefix, - rt.resources.output_prefix) and - not options.keep_stage_files): - printer.error('stage and output refer to the same directory; ' - 'if this is on purpose, please use also the ' - "`--keep-stage-files' option.") + if (os_ext.samefile(rt.stage_prefix, rt.output_prefix) and + not site_config.get('general/0/keep_stage_files')): + printer.error("stage and output refer to the same directory; " + "if this is on purpose, please use the " + "'--keep-stage-files' option.") sys.exit(1) - if options.timestamp: - rt.resources.timefmt = options.timestamp - - # Configure performance logging - # NOTE: we need resources to be configured in order to set the global - # perf. logging prefix correctly - if options.perflogdir: - rt.resources.perflogdir = os_ext.expandvars(options.perflogdir) - - logging.LOG_CONFIG_OPTS['handlers.filelog.prefix'] = (rt.resources. - perflog_prefix) - # Show configuration after everything is set up - if options.show_config: - printer.info(rt.show_config()) - sys.exit(0) - - if options.show_config_env: - envname = options.show_config_env - for p in rt.system.partitions: - environ = p.environment(envname) - if environ: - break - - if environ is None: - printer.error('no such environment: ' + envname) - sys.exit(1) + if options.show_config_param: + config_param = options.show_config_param + if config_param == 'all': + printer.info(str(rt.site_config)) + else: + value = rt.get_option(config_param) + if value is None: + printer.error( + f'no such configuration parameter found: {config_param}' + ) + else: + printer.info(json.dumps(value, indent=2)) - printer.info(environ.details()) sys.exit(0) - if hasattr(settings, 'perf_logging_config'): - try: - logging.configure_perflogging(settings.perf_logging_config) - except (OSError, ConfigError) as e: - printer.error('could not configure performance logging: %s\n' % e) - sys.exit(1) - else: - printer.warning('no performance logging is configured; ' - 'please check documentation') - # Setup the check loader - if options.checkpath: - load_path = [] - for d in options.checkpath.split(':'): - d = os_ext.expandvars(d) - if not os.path.exists(d): - printer.warning("%s: path `%s' does not exist. Skipping..." % - (argparser.prog, d)) - continue - - load_path.append(os.path.realpath(d)) - - load_path = os_ext.unique_abs_paths(load_path, - prune_children=options.recursive) - loader = RegressionCheckLoader( - load_path, recurse=options.recursive, - ignore_conflicts=options.ignore_check_conflicts) - else: - loader = RegressionCheckLoader( - load_path=settings.checks_path, - prefix=reframe.INSTALL_PREFIX, - recurse=settings.checks_path_recurse) - + loader = RegressionCheckLoader( + load_path=site_config.get('general/0/check_search_path'), + recurse=site_config.get('general/0/check_search_recursive'), + ignore_conflicts=site_config.get('general/0/ignore_check_conflicts') + ) printer.debug(argparse.format_options(options)) - if options.no_deprecation_warnings: - warnings.filterwarnings('ignore', category=ReframeDeprecationWarning) + def print_infoline(param, value): + param = param + ':' + printer.info(f" {param.ljust(18)} {value}") # Print command line - printer.info('Command line: %s' % ' '.join(sys.argv)) - printer.info('Reframe version: ' + os_ext.reframe_version()) - printer.info('Launched by user: ' + (os_ext.osuser() or '')) - printer.info('Launched on host: ' + socket.gethostname()) - - # Print important paths - printer.info('Reframe paths') - printer.info('=============') - printer.info(' Check prefix : %s' % loader.prefix) - printer.info('%03s Check search path : %s' % - ('(R)' if loader.recurse else '', - "'%s'" % ':'.join(loader.load_path))) - printer.info(' Current working dir : %s' % os.getcwd()) - printer.info(' Stage dir prefix : %s' % rt.resources.stage_prefix) - printer.info(' Output dir prefix : %s' % rt.resources.output_prefix) - printer.info( - ' Perf. logging prefix : %s' % - os.path.abspath(logging.LOG_CONFIG_OPTS['handlers.filelog.prefix'])) + printer.info(f"[ReFrame Setup]") + print_infoline('version', os_ext.reframe_version()) + print_infoline('command', repr(' '.join(sys.argv))) + print_infoline('launched by', + f"{os_ext.osuser() or ''}@{socket.gethostname()}") + print_infoline('working directory', repr(os.getcwd())) + print_infoline('check search path', + f"{'(R)' if loader.recurse else ''} " + f"{':'.join(loader.load_path)!r}") + print_infoline('stage directory', repr(rt.stage_prefix)) + print_infoline('output directory', repr(rt.output_prefix)) + print_infoline('performance logs', repr(rt.perflogdir)) + printer.info('') try: # Locate and load checks try: @@ -531,29 +502,23 @@ def main(): dependency.validate_deps(testgraph) testcases = dependency.toposort(testgraph) - # Unload regression's module and load user-specified modules - if hasattr(settings, 'reframe_module'): - printer.warning( - "the 'reframe_module' configuration option will be ignored; " - "please use the '-u' or '--unload-module' options" - ) - - if options.purge_env: + # Manipulate ReFrame's environment + if site_config.get('general/0/purge_environment'): rt.modules_system.unload_all() else: - for m in options.unload_modules: + for m in site_config.get('general/0/unload_modules'): rt.modules_system.unload_module(m) # Load the environment for the current system try: - env.load(rt.system.preload_environ) + runtime.loadenv(rt.system.preload_environ) except EnvironError as e: printer.error("failed to load current system's environment; " "please check your configuration") printer.debug(str(e)) raise - for m in options.user_modules: + for m in site_config.get('general/0/user_modules'): try: rt.modules_system.load_module(m, force=True) except EnvironError as e: @@ -596,8 +561,9 @@ def main(): exec_policy.strict_check = options.strict exec_policy.skip_sanity_check = options.skip_sanity_check exec_policy.skip_performance_check = options.skip_performance_check - exec_policy.keep_stage_files = options.keep_stage_files - + exec_policy.keep_stage_files = site_config.get( + 'general/0/keep_stage_files' + ) try: errmsg = "invalid option for --flex-alloc-nodes: '{0}'" sched_flex_alloc_nodes = int(options.flex_alloc_nodes) @@ -653,7 +619,7 @@ def main(): sys.exit(0) - except (KeyboardInterrupt, ReframeForceExitError): + except KeyboardInterrupt: sys.exit(1) except ReframeError as e: printer.error(str(e)) @@ -663,8 +629,8 @@ def main(): sys.exit(1) finally: try: - if options.save_log_files: - logging.save_log_files(rt.resources.output_prefix) + if site_config.get('general/0/save_log_files'): + logging.save_log_files(rt.output_prefix) except OSError as e: printer.error('could not save log file: %s' % e) diff --git a/reframe/frontend/executors/__init__.py b/reframe/frontend/executors/__init__.py index 7291916cbd..34344c2848 100644 --- a/reframe/frontend/executors/__init__.py +++ b/reframe/frontend/executors/__init__.py @@ -277,6 +277,10 @@ def __init__(self, policy, printer=None, max_retries=0): def __repr__(self): return debug.repr(self) + @property + def max_retries(self): + return self._max_retries + @property def policy(self): return self._policy @@ -398,6 +402,10 @@ def exit(self): def runcase(self, case): '''Run a test case.''' + # Pick the right subconfig for the current partition + rt = runtime.runtime() + _, partition, _ = case + rt.site_config.select_subconfig(partition.fullname) if self.strict_check: case.check.strict_check = True diff --git a/reframe/frontend/loader.py b/reframe/frontend/loader.py index 15300404f6..ccdff1474a 100644 --- a/reframe/frontend/loader.py +++ b/reframe/frontend/loader.py @@ -13,6 +13,7 @@ import reframe.core.debug as debug import reframe.utility as util +import reframe.utility.os_ext as os_ext from reframe.core.exceptions import NameConflictError, RegressionTestLoadError from reframe.core.logging import getlogger @@ -37,10 +38,10 @@ def visit_ImportFrom(self, node): class RegressionCheckLoader: - def __init__(self, load_path, prefix='', - recurse=False, ignore_conflicts=False): - self._load_path = load_path - self._prefix = prefix or '' + def __init__(self, load_path, recurse=False, ignore_conflicts=False): + # Expand any environment variables and symlinks + load_path = [os.path.realpath(os_ext.expandvars(p)) for p in load_path] + self._load_path = os_ext.unique_abs_paths(load_path, recurse) self._recurse = recurse self._ignore_conflicts = ignore_conflicts @@ -156,7 +157,6 @@ def load_all(self): If a prefix exists, it will be prepended to each path.''' checks = [] for d in self._load_path: - d = os.path.join(self._prefix, d) if not os.path.exists(d): continue if os.path.isdir(d): diff --git a/reframe/settings.py b/reframe/settings.py deleted file mode 100644 index 811d78d3d2..0000000000 --- a/reframe/settings.py +++ /dev/null @@ -1,104 +0,0 @@ -# Copyright 2016-2020 Swiss National Supercomputing Centre (CSCS/ETH Zurich) -# ReFrame Project Developers. See the top-level LICENSE file for details. -# -# SPDX-License-Identifier: BSD-3-Clause - -# -# ReFrame generic settings -# - - -class ReframeSettings: - job_poll_intervals = [1, 2, 3] - job_submit_timeout = 60 - checks_path = ['checks/'] - checks_path_recurse = True - site_configuration = { - 'systems': { - 'generic': { - 'descr': 'Generic example system', - 'hostnames': ['localhost'], - 'partitions': { - 'login': { - 'scheduler': 'local', - 'modules': [], - 'access': [], - 'environs': ['builtin-gcc'], - 'descr': 'Login nodes' - } - } - } - }, - - 'environments': { - '*': { - 'builtin': { - 'type': 'ProgEnvironment', - 'cc': 'cc', - 'cxx': '', - 'ftn': '', - }, - - 'builtin-gcc': { - 'type': 'ProgEnvironment', - 'cc': 'gcc', - 'cxx': 'g++', - 'ftn': 'gfortran', - } - } - } - } - - logging_config = { - 'level': 'DEBUG', - 'handlers': [ - { - 'type': 'file', - 'name': 'reframe.log', - 'level': 'DEBUG', - 'format': '[%(asctime)s] %(levelname)s: ' - '%(check_info)s: %(message)s', - 'append': False, - }, - - # Output handling - { - 'type': 'stream', - 'name': 'stdout', - 'level': 'INFO', - 'format': '%(message)s' - }, - { - 'type': 'file', - 'name': 'reframe.out', - 'level': 'INFO', - 'format': '%(message)s', - 'append': False, - } - ] - } - - perf_logging_config = { - 'level': 'DEBUG', - 'handlers': [ - { - 'type': 'filelog', - 'prefix': '%(check_system)s/%(check_partition)s', - 'level': 'INFO', - 'format': ( - '%(check_job_completion_time)s|reframe %(version)s|' - '%(check_info)s|jobid=%(check_jobid)s|' - '%(check_perf_var)s=%(check_perf_value)s|' - 'ref=%(check_perf_ref)s ' - '(l=%(check_perf_lower_thres)s, ' - 'u=%(check_perf_upper_thres)s)|' - '%(check_perf_unit)s' - ), - 'datefmt': '%FT%T%:z', - 'append': True - } - ] - } - - -settings = ReframeSettings() diff --git a/schemas/config.json b/schemas/config.json index 6378f2232d..d5f09e21e8 100644 --- a/schemas/config.json +++ b/schemas/config.json @@ -37,7 +37,10 @@ "handler_common": { "type": "object", "properties": { - "type": {"type": "string"}, + "type": { + "type": "string", + "enum": ["file", "filelog", "graylog", "stream", "syslog"] + }, "level": {"$ref": "#/defs/loglevel"}, "format": {"type": "string"}, "datefmt": {"type": "string"} @@ -67,8 +70,11 @@ {"$ref": "#/defs/handler_common"}, { "properties": { - "prefix": {"type": "string"} - } + "basedir": {"type": "string"}, + "prefix": {"type": "string"}, + "append": {"type": "boolean"} + }, + "required": ["prefix"] } ] }, @@ -106,7 +112,17 @@ "type": "string", "enum": ["tcp", "udp"] }, - "facility": {"type": "string"}, + "facility": {"type": "string", + "enum": ["auth", "authpriv", + "cron", "daemon", + "ftp", "kern", + "lpr", "mail", + "news", "syslog", + "user", "uucp", + "local0", "local1", + "local2", "local3", + "local4", "local5", + "local6", "local7"]}, "address": {"type": "string"} }, "required": ["address"] @@ -129,14 +145,14 @@ }, "modules_system": { "type": "string", - "enum": ["tmod", "tmod31", "tmod32", "tmod4", "lmod"] + "enum": ["tmod", "tmod31", "tmod32", + "tmod4", "lmod", "nomod"] }, "modules": {"$ref": "#/defs/modules_list"}, "variables": {"$ref": "#/defs/envvar_list"}, "prefix": {"type": "string"}, "stagedir": {"type": "string"}, "outputdir": {"type": "string"}, - "perflogdir": {"type": "string"}, "resourcesdir": {"type": "string"}, "partitions": { "type": "array", @@ -169,10 +185,12 @@ "items": { "type": "object", "properties": { - "name": { + "type": { "type": "string", - "enum": ["Docker", "Sarus", - "Singularity"] + "enum": [ + "Docker", "Sarus", + "Shifter", "Singularity" + ] }, "modules": { "$ref": "#/defs/modules_list" @@ -181,7 +199,7 @@ "$ref": "#/defs/envvar_list" } }, - "required": ["name"] + "required": ["type"] } }, "modules": {"$ref": "#/defs/modules_list"}, @@ -196,18 +214,20 @@ "options": { "type": "array", "items": {"type": "string"} - }, - "additionalProperties": false - } + } + }, + "required": ["name"], + "additionalProperties": false } } }, "required": ["name", "scheduler", "launcher"], "additionalProperties": false - } + }, + "minItems": 1 } }, - "required": ["name"], + "required": ["name", "hostnames", "partitions"], "additionalProperties": false } }, @@ -277,20 +297,10 @@ {"$ref": "#/defs/stream_handler"}, {"$ref": "#/defs/syslog_handler"} ] - } + }, + "minItems": 1 }, - "target_systems": {"$ref": "#/defs/system_ref"} - }, - "additionalProperties": false - } - }, - "perf_logging": { - "type": "array", - "items": { - "type": "object", - "properties": { - "level": {"$ref": "#/defs/loglevel"}, - "handlers": { + "handlers_perflog": { "type": "array", "items": { "anyOf": [ @@ -304,6 +314,7 @@ }, "target_systems": {"$ref": "#/defs/system_ref"} }, + "required": ["handlers", "handlers_perflog"], "additionalProperties": false } }, @@ -333,35 +344,98 @@ "items": {"type": "string"} }, "check_search_recursive": {"type": "boolean"}, - "target_systems": {"$ref": "#/defs/system_ref"} + "colorize": {"type": "boolean"}, + "ignore_check_conflicts": {"type": "boolean"}, + "keep_stage_files": {"type": "boolean"}, + "module_map_file": {"type": "string"}, + "module_mappings": { + "type": "array", + "items": {"type": "string"} + }, + "non_default_craype": {"type": "boolean"}, + "purge_environment": {"type": "boolean"}, + "save_log_files": {"type": "boolean"}, + "target_systems": {"$ref": "#/defs/system_ref"}, + "timestamp_dirs": {"type": "string"}, + "unload_modules": { + "type": "array", + "items": {"type": "string"} + }, + "user_modules": { + "type": "array", + "items": {"type": "string"} + }, + "verbose": {"type": "number"} }, "additionalProperties": false } } }, - "required": ["systems", "environments", "logging", "perf_logging"], + "required": ["systems", "environments", "logging"], "additionalProperties": false, "defaults": { + "environments/modules": [], + "environments/variables": [], "environments/cc": "cc", "environments/cxx": "CC", "environments/ftn": "ftn", + "environments/cppflags": [], + "environments/cflags": [], + "environments/cxxflags": [], + "environments/fflags": [], + "environments/ldflags": [], "environments/target_systems": ["*"], - "general/check_search_path": ["checks/"], - "general/check_search_recursive": "true", + "general/check_search_path": ["${RFM_INSTALL_PREFIX}/checks/"], + "general/check_search_recursive": true, + "general/colorize": true, + "general/ignore_check_conflicts": false, + "general/keep_stage_files": false, + "general/module_map_file": "", + "general/module_mappings": [], + "general/non_default_craype": false, + "general/purge_environment": false, + "general/save_log_files": false, "general/target_systems": ["*"], - "perf_logging/target_systems": ["*"], - "logging/handlers/level": "debug", - "logging/handlers/file/append": false, - "logging/handlers/file/timestamp": false, - "logging/handlers/stream/name": "stdout", - "logging/handlers/syslog/socktype": "udp", - "logging/handlers/syslog/facility": "user", - "logging/level": "info", + "general/timestamp_dirs": "", + "general/unload_modules": [], + "general/user_modules": [], + "general/verbose": 0, + "logging/level": "debug", "logging/target_systems": ["*"], + "logging/handlers*/*_level": "info", + "logging/handlers*/*_format": "%(message)s", + "logging/handlers*/*_datefmt": "%FT%T", + "logging/handlers*/file_append": false, + "logging/handlers*/file_timestamp": false, + "logging/handlers*/filelog_append": true, + "logging/handlers*/filelog_basedir": "./perflogs", + "logging/handlers*/graylog_extras": {}, + "logging/handlers*/stream_name": "stdout", + "logging/handlers*/syslog_socktype": "udp", + "logging/handlers*/syslog_facility": "user", + "modes/options": [], "modes/target_systems": ["*"], "schedulers/job_submit_timeout": 60, "schedulers/target_systems": ["*"], + "systems/descr": "", + "systems/modules_system": "nomod", + "systems/modules": [], + "systems/variables": [], "systems/prefix": ".", - "systems/resourcesdir": "." + "systems/outputdir": "", + "systems/resourcesdir": ".", + "systems/stagedir": "", + "systems/partitions/descr": "", + "systems/partitions/access": [], + "systems/partitions/environs": [], + "systems/partitions/container_platforms": [], + "systems/partitions/container_platforms/modules": [], + "systems/partitions/container_platforms/variables": [], + "systems/partitions/resources": [], + "systems/partitions/resources/options": [], + "systems/partitions/modules": [], + "systems/partitions/variables": [], + "systems/partitions/environs": [], + "systems/partitions/max_jobs": 1 } } diff --git a/schemas/settings.py b/schemas/settings.py deleted file mode 100644 index 0f2d1b7c0c..0000000000 --- a/schemas/settings.py +++ /dev/null @@ -1,183 +0,0 @@ -# Copyright 2016-2020 Swiss National Supercomputing Centre (CSCS/ETH Zurich) -# ReFrame Project Developers. See the top-level LICENSE file for details. -# -# SPDX-License-Identifier: BSD-3-Clause - -# -# New style configuration -# -# This corresponds to the current unittests/resources/settings.py -# - -site_configuration = { - 'systems': [ - { - 'name': 'generic', - 'descr': 'Generic example system', - 'hostnames': ['localhost'], - 'partitions': [ - { - 'name': 'login', - 'descr': 'Login nodes', - 'scheduler': 'local', - 'launcher': 'local', - 'environs': ['builtin-gcc'] - } - ] - }, - { - 'name': 'testsys', - 'descr': 'Fake system for unit tests', - 'hostnames': ['testsys'], - 'prefix': '.rfm_testing', - 'resourcesdir': '.rfm_testing/resources', - 'perflogdir': '.rfm_testing/perflogs', - 'modules': ['foo/1.0'], - 'variables': [['FOO_CMD', 'foobar']], - 'partitions': [ - { - 'name': 'login', - 'scheduler': 'local', - 'launcher': 'local', - 'environs': ['PrgEnv-cray', 'PrgEnv-gnu', 'builtin-gcc'], - 'descr': 'Login nodes' - }, - { - 'name': 'gpu', - 'descr': 'GPU partition', - 'scheduler': 'slurm', - 'launcher': 'srun', - 'modules': ['foogpu'], - 'variables': [['FOO_GPU', 'yes']], - 'resources': [ - { - 'name': 'gpu', - 'options': ['--gres=gpu:{num_gpus_per_node}'], - }, - { - 'name': 'datawarp', - 'options': [ - '#DW jobdw capacity={capacity}', - '#DW stage_in source={stagein_src}' - ] - } - ], - 'environs': ['PrgEnv-gnu', 'builtin-gcc'], - } - ] - }, - { - 'name': 'sys0', - 'descr': 'System for checking test dependencies', - 'hostnames': [r'sys\d+'], - 'partitions': [ - { - 'name': 'p0', - 'scheduler': 'local', - 'launcher': 'local', - 'environs': ['e0', 'e1'] - }, - { - 'name': 'p1', - 'scheduler': 'local', - 'launcher': 'local', - 'environs': ['e0', 'e1'] - } - - ] - } - ], - 'environments': [ - { - 'name': 'PrgEnv-gnu', - 'modules': ['PrgEnv-gnu'], - 'cc': 'gcc', - 'cxx': 'g++', - 'ftn': 'gfortran', - 'target_systems': ['testsys:login'] - }, - { - 'name': 'PrgEnv-gnu', - 'modules': ['PrgEnv-gnu'], - }, - { - 'name': 'PrgEnv-cray', - 'modules': ['PrgEnv-cray'], - }, - { - 'name': 'builtin', - 'cc': 'cc', - 'cxx': '', - 'ftn': '' - }, - { - 'name': 'builtin-gcc', - 'cc': 'gcc', - 'cxx': 'g++', - 'ftn': 'gfortran' - }, - { - 'name': 'e0', - 'modules': ['m0'] - }, - { - 'name': 'e1', - 'modules': ['m1'] - } - ], - 'modes': [ - { - 'name': 'unittest', - 'options': [ - '-c', 'unittests/resources/checks/hellocheck.py', - '-p', 'builtin-gcc', - '--force-local' - ] - } - ], - 'logging': [ - { - 'level': 'debug', - 'handlers': [ - { - 'type': 'file', - 'name': '.rfm_unittest.log', - 'level': 'debug', - 'format': ('[%(asctime)s] %(levelname)s: ' - '%(check_name)s: %(message)s'), - 'datefmt': '%FT%T', - 'append': False, - }, - { - 'type': 'stream', - 'name': 'stdout', - 'level': 'info', - 'format': '%(message)s' - } - ] - } - ], - 'perf_logging': [ - { - 'level': 'debug', - 'handlers': [ - { - 'type': 'filelog', - 'prefix': '%(check_system)s/%(check_partition)s', - 'level': 'info', - 'format': ( - '%(check_job_completion_time)s|reframe %(version)s|' - '%(check_info)s|jobid=%(check_jobid)s|' - '%(check_perf_var)s=%(check_perf_value)s|' - 'ref=%(check_perf_ref)s ' - '(l=%(check_perf_lower_thres)s, ' - 'u=%(check_perf_upper_thres)s)|' - '%(check_perf_unit)s' - ), - 'datefmt': '%FT%T%:z', - 'append': True - } - ] - } - ] -} diff --git a/test_reframe.py b/test_reframe.py index ad3a309ade..69aa9046e5 100755 --- a/test_reframe.py +++ b/test_reframe.py @@ -16,16 +16,20 @@ parser = argparse.ArgumentParser( add_help=False, usage='%(prog)s [REFRAME_OPTIONS...] [NOSE_OPTIONS...]') - parser.add_argument('--rfm-user-config', action='store', metavar='FILE', - help='Config file to use for native unit tests.') - parser.add_argument('--rfm-help', action='help', - help='Print this help message and exit.') - + parser.add_argument( + '--rfm-user-config', action='store', metavar='FILE', + help='Config file to use for native unit tests.' + ) + parser.add_argument( + '--rfm-user-system', action='store', metavar='NAME', + help="Specific system to use from user's configuration" + ) + parser.add_argument( + '--rfm-help', action='help', help='Print this help message and exit.' + ) options, rem_args = parser.parse_known_args() - if options.rfm_user_config: - fixtures.set_user_config(options.rfm_user_config) - + fixtures.USER_CONFIG_FILE = options.rfm_user_config + fixtures.USER_SYSTEM = options.rfm_user_system fixtures.init_runtime() - sys.argv = [sys.argv[0], *rem_args] sys.exit(pytest.main()) diff --git a/tools/convert_config.py b/tools/convert_config.py index 397b3dba6b..04674cc866 100644 --- a/tools/convert_config.py +++ b/tools/convert_config.py @@ -28,5 +28,5 @@ print( f"Conversion successful! " - f"Please find the converted file at '{new_config}'." + f"The converted file can be found at '{new_config}'." ) diff --git a/tutorial/config/settings.py b/tutorial/config/settings.py index a3ccb024ca..a4333e4f09 100644 --- a/tutorial/config/settings.py +++ b/tutorial/config/settings.py @@ -4,134 +4,134 @@ # SPDX-License-Identifier: BSD-3-Clause # -# Minimal settings for ReFrame tutorial on Piz Daint +# Minimal settings for ReFrame's tutorial for running on Piz Daint # - -class ReframeSettings: - job_poll_intervals = [1, 2, 3] - job_submit_timeout = 60 - checks_path = ['checks/'] - checks_path_recurse = True - site_configuration = { - 'systems': { - 'daint': { - 'descr': 'Piz Daint', - 'hostnames': ['daint'], - 'modules_system': 'tmod', - 'partitions': { - 'login': { - 'scheduler': 'local', - 'modules': [], - 'access': [], - 'environs': ['PrgEnv-cray', 'PrgEnv-gnu', - 'PrgEnv-intel', 'PrgEnv-pgi'], - 'descr': 'Login nodes', - 'max_jobs': 4 - }, - - 'gpu': { - 'scheduler': 'nativeslurm', - 'modules': ['daint-gpu'], - 'access': ['--constraint=gpu'], - 'environs': ['PrgEnv-cray', 'PrgEnv-gnu', - 'PrgEnv-intel', 'PrgEnv-pgi'], - 'container_platforms': { - 'Singularity': { - 'modules': ['Singularity'] - } - }, - 'descr': 'Hybrid nodes (Haswell/P100)', - 'max_jobs': 100 - }, - - 'mc': { - 'scheduler': 'nativeslurm', - 'modules': ['daint-mc'], - 'access': ['--constraint=mc'], - 'environs': ['PrgEnv-cray', 'PrgEnv-gnu', - 'PrgEnv-intel', 'PrgEnv-pgi'], - 'container_platforms': { - 'Singularity': { - 'modules': ['Singularity'] - } - }, - 'descr': 'Multicore nodes (Broadwell)', - 'max_jobs': 100 - } +site_configuration = { + 'systems': [ + { + 'name': 'daint', + 'descr': 'Piz Daint', + 'hostnames': ['daint'], + 'modules_system': 'tmod', + 'partitions': [ + { + 'name': 'login', + 'descr': 'Login nodes', + 'scheduler': 'local', + 'launcher': 'local', + 'environs': [ + 'PrgEnv-cray', + 'PrgEnv-gnu', + 'PrgEnv-intel', + 'PrgEnv-pgi' + ], + 'max_jobs': 4, + }, + { + 'name': 'gpu', + 'descr': 'Hybrid nodes (Haswell/P100)', + 'scheduler': 'slurm', + 'launcher': 'srun', + 'modules': ['daint-gpu'], + 'access': ['--constraint=gpu'], + 'environs': [ + 'PrgEnv-cray', + 'PrgEnv-gnu', + 'PrgEnv-intel', + 'PrgEnv-pgi' + ], + 'container_platforms': [ + { + 'name': 'Singularity', + 'modules': ['Singularity'] + } + ], + 'max_jobs': 100, + }, + { + 'name': 'mc', + 'descr': 'Multicore nodes (Broadwell)', + 'scheduler': 'slurm', + 'launcher': 'srun', + 'modules': ['daint-mc'], + 'access': ['--constraint=mc'], + 'environs': [ + 'PrgEnv-cray', + 'PrgEnv-gnu', + 'PrgEnv-intel', + 'PrgEnv-pgi' + ], + 'container_platforms': [ + { + 'name': 'Singularity', + 'modules': ['Singularity'] + } + ], + 'max_jobs': 100, } - } + ] + } + ], + 'environments': [ + { + 'name': 'PrgEnv-cray', + 'modules': ['PrgEnv-cray'] }, - - 'environments': { - '*': { - 'PrgEnv-cray': { - 'modules': ['PrgEnv-cray'], - }, - 'PrgEnv-gnu': { - 'modules': ['PrgEnv-gnu'], + { + 'name': 'PrgEnv-gnu', + 'modules': ['PrgEnv-gnu'] + }, + { + 'name': 'PrgEnv-intel', + 'modules': ['PrgEnv-intel'] + }, + { + 'name': 'PrgEnv-pgi', + 'modules': ['PrgEnv-pgi'] + } + ], + 'logging': [ + { + 'level': 'debug', + 'handlers': [ + { + 'type': 'file', + 'name': 'reframe.log', + 'level': 'debug', + 'format': '[%(asctime)s] %(levelname)s: %(check_name)s: %(message)s', # noqa: E501 + 'append': False }, - - 'PrgEnv-intel': { - 'modules': ['PrgEnv-intel'], + { + 'type': 'stream', + 'name': 'stdout', + 'level': 'info', + 'format': '%(message)s' }, - - 'PrgEnv-pgi': { - 'modules': ['PrgEnv-pgi'], + { + 'type': 'file', + 'name': 'reframe.out', + 'level': 'info', + 'format': '%(message)s', + 'append': False } - } + ], + 'handlers_perflog': [ + { + 'type': 'filelog', + 'prefix': '%(check_system)s/%(check_partition)s', + 'level': 'info', + 'format': '%(check_job_completion_time)s|reframe %(version)s|%(check_info)s|jobid=%(check_jobid)s|%(check_perf_var)s=%(check_perf_value)s|ref=%(check_perf_ref)s (l=%(check_perf_lower_thres)s, u=%(check_perf_upper_thres)s)', # noqa: E501 + 'datefmt': '%FT%T%:z', + 'append': True + } + ] } - } - - logging_config = { - 'level': 'DEBUG', - 'handlers': [ - { - 'type': 'file', - 'name': 'reframe.log', - 'level': 'DEBUG', - 'format': '[%(asctime)s] %(levelname)s: ' - '%(check_name)s: %(message)s', - 'append': False, - }, - - # Output handling - { - 'type': 'stream', - 'name': 'stdout', - 'level': 'INFO', - 'format': '%(message)s' - }, - { - 'type': 'file', - 'name': 'reframe.out', - 'level': 'INFO', - 'format': '%(message)s', - 'append': False, - } - ] - } - - perf_logging_config = { - 'level': 'DEBUG', - 'handlers': [ - { - 'type': 'filelog', - 'prefix': '%(check_system)s/%(check_partition)s', - 'level': 'INFO', - 'format': ( - '%(check_job_completion_time)s|reframe %(version)s|' - '%(check_info)s|jobid=%(check_jobid)s|' - '%(check_perf_var)s=%(check_perf_value)s|' - 'ref=%(check_perf_ref)s ' - '(l=%(check_perf_lower_thres)s, ' - 'u=%(check_perf_upper_thres)s)' - ), - 'datefmt': '%FT%T%:z', - 'append': True - } - ] - } - - -settings = ReframeSettings() + ], + 'general': [ + { + 'check_search_path': ['tutorial/'], + 'check_search_recursive': True + } + ] +} diff --git a/unittests/fixtures.py b/unittests/fixtures.py index a3c3724367..334813831b 100644 --- a/unittests/fixtures.py +++ b/unittests/fixtures.py @@ -9,41 +9,33 @@ import os import tempfile +import reframe import reframe.core.config as config import reframe.core.modules as modules import reframe.core.runtime as rt -from reframe.core.exceptions import UnknownSystemError +import reframe.utility.os_ext as os_ext TEST_RESOURCES = os.path.join( - os.path.dirname(os.path.realpath(__file__)), 'resources') + os.path.dirname(os.path.realpath(__file__)), 'resources' +) TEST_RESOURCES_CHECKS = os.path.join(TEST_RESOURCES, 'checks') TEST_MODULES = os.path.join( - os.path.dirname(os.path.realpath(__file__)), 'modules') + os.path.dirname(os.path.realpath(__file__)), 'modules' +) # Unit tests site configuration -TEST_SITE_CONFIG = None +TEST_CONFIG_FILE = 'unittests/resources/settings.py' # User supplied configuration file and site configuration USER_CONFIG_FILE = None -USER_SITE_CONFIG = None - - -def set_user_config(config_file): - global USER_CONFIG_FILE, USER_SITE_CONFIG - - USER_CONFIG_FILE = config_file - user_settings = config.load_settings_from_file(config_file) - USER_SITE_CONFIG = user_settings.site_configuration +USER_SYSTEM = None def init_runtime(): - global TEST_SITE_CONFIG - - settings = config.load_settings_from_file( - 'unittests/resources/settings.py') - TEST_SITE_CONFIG = settings.site_configuration - rt.init_runtime(TEST_SITE_CONFIG, 'generic') + site_config = config.load_config('unittests/resources/settings.py') + site_config.select_subconfig('generic') + rt.init_runtime(site_config) def switch_to_user_runtime(fn): @@ -52,28 +44,22 @@ def switch_to_user_runtime(fn): If no such configuration exists, this decorator returns the target function untouched. ''' - if USER_SITE_CONFIG is None: + if USER_CONFIG_FILE is None: return fn - return rt.switch_runtime(USER_SITE_CONFIG)(fn) + return rt.switch_runtime(USER_CONFIG_FILE, USER_SYSTEM)(fn) -# FIXME: This may conflict in the unlikely situation that a user defines a -# system named `kesch` with a partition named `pn`. -def partition_with_scheduler(name=None, skip_partitions=['kesch:pn']): +def partition_by_scheduler(name=None): '''Retrieve a system partition from the runtime whose scheduler is registered with ``name``. If ``name`` is :class:`None`, any partition with a non-local scheduler will be returned. - Partitions specified in ``skip_partitions`` will be skipped from searching. ''' system = rt.runtime().system for p in system.partitions: - if p.fullname in skip_partitions: - continue - if name is None and not p.scheduler.is_local: return p @@ -83,6 +69,22 @@ def partition_with_scheduler(name=None, skip_partitions=['kesch:pn']): return None +def partition_by_name(name): + for p in rt.runtime().system.partitions: + if p.name == name: + return p + + return None + + +def environment_by_name(name, partition): + for e in partition.environs: + if e.name == name: + return e + + return None + + def has_sane_modules_system(): return not isinstance(rt.runtime().modules_system.backend, modules.NoModImpl) @@ -99,3 +101,19 @@ def _set_prefix(cls): return cls return _set_prefix + + +def safe_rmtree(path, **kwargs): + '''Do some safety checks before removing path to protect against silly, but + catastrophic bugs, during development. + + Do not allow removing any subdirectory of reframe or any directory + containing reframe. Also do not allow removing the user's $HOME directory. + ''' + + path = os.path.abspath(path) + common_path = os.path.commonpath([reframe.INSTALL_PREFIX, path]) + assert common_path != reframe.INSTALL_PREFIX + assert common_path != path + assert path != os.environ['HOME'] + os_ext.rmtree(path, **kwargs) diff --git a/unittests/resources/checks/frontend_checks.py b/unittests/resources/checks/frontend_checks.py index 17331036f3..1e7b77d2c2 100644 --- a/unittests/resources/checks/frontend_checks.py +++ b/unittests/resources/checks/frontend_checks.py @@ -7,6 +7,10 @@ # Special checks for testing the front-end # +import os +import signal +import time + import reframe as rfm import reframe.utility.sanity as sn from reframe.core.exceptions import ReframeError, PerformanceError @@ -32,6 +36,7 @@ def __init__(self): def raise_error(self): raise ReframeError('Setup failure') + @rfm.simple_test class BadSetupCheckEarly(BaseFrontendCheck): def __init__(self): @@ -199,3 +204,19 @@ def __init__(self, run_to_pass, filename): self.post_run = ['((current_run++))', 'echo $current_run > %s' % filename] self.sanity_patterns = sn.assert_found('%d' % run_to_pass, self.stdout) + + +class SelfKillCheck(rfm.RunOnlyRegressionTest, special=True): + def __init__(self): + self.local = True + self.valid_systems = ['*'] + self.valid_prog_environs = ['*'] + self.executable = 'echo hello' + self.sanity_patterns = sn.assert_found('hello', self.stdout) + self.tags = {type(self).__name__} + self.maintainers = ['TM'] + + def run(self): + super().run() + time.sleep(0.5) + os.kill(os.getpid(), signal.SIGTERM) diff --git a/unittests/resources/checks_unlisted/selfkill.py b/unittests/resources/checks_unlisted/selfkill.py deleted file mode 100644 index 5995ec6f17..0000000000 --- a/unittests/resources/checks_unlisted/selfkill.py +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright 2016-2020 Swiss National Supercomputing Centre (CSCS/ETH Zurich) -# ReFrame Project Developers. See the top-level LICENSE file for details. -# -# SPDX-License-Identifier: BSD-3-Clause - -# -# Check for testing handling of the TERM signal -# -import os -import signal -import time - -import reframe as rfm -import reframe.utility.sanity as sn - - -@rfm.simple_test -class SelfKillCheck(rfm.RunOnlyRegressionTest, special=True): - def __init__(self): - self.local = True - self.valid_systems = ['*'] - self.valid_prog_environs = ['*'] - self.executable = 'echo hello' - self.sanity_patterns = sn.assert_found('hello', self.stdout) - self.tags = {type(self).__name__} - self.maintainers = ['TM'] - - def run(self): - super().run() - time.sleep(0.5) - os.kill(os.getpid(), signal.SIGTERM) diff --git a/unittests/resources/settings.py b/unittests/resources/settings.py index d3ddc060a7..b1361ef429 100644 --- a/unittests/resources/settings.py +++ b/unittests/resources/settings.py @@ -4,169 +4,188 @@ # SPDX-License-Identifier: BSD-3-Clause # -# ReFrame settings for use in the unit tests +# Configuration file just for unit testing # - -class ReframeSettings: - job_poll_intervals = [1, 2, 3] - job_submit_timeout = 60 - checks_path = ['checks/'] - checks_path_recurse = True - site_configuration = { - 'systems': { - # Generic system configuration that allows to run ReFrame locally - # on any system. - 'generic': { - 'descr': 'Generic example system', - 'hostnames': ['localhost'], - 'partitions': { - 'login': { - 'scheduler': 'local', - 'modules': [], - 'access': [], - 'environs': ['builtin-gcc'], - 'descr': 'Login nodes' - }, +site_configuration = { + 'systems': [ + { + 'name': 'generic', + 'descr': 'Generic example system', + 'hostnames': ['.*'], + 'partitions': [ + { + 'name': 'default', + 'descr': 'Login nodes', + 'scheduler': 'local', + 'launcher': 'local', + 'environs': ['builtin-gcc'] } - }, - 'testsys': { - # A fake system simulating a possible cluster configuration, in - # order to test different aspects of the framework. - 'descr': 'Fake system for unit tests', - 'hostnames': ['testsys'], - 'prefix': '.rfm_testing', - 'resourcesdir': '.rfm_testing/resources', - 'perflogdir': '.rfm_testing/perflogs', - 'modules': ['foo/1.0'], - 'variables': {'FOO_CMD': 'foobar'}, - 'partitions': { - 'login': { - 'scheduler': 'local', - 'resources': {}, - 'environs': ['PrgEnv-cray', 'PrgEnv-gnu', 'builtin-gcc'], - 'descr': 'Login nodes' - }, - 'gpu': { - 'scheduler': 'nativeslurm', - 'modules': ['foogpu'], - 'variables': {'FOO_GPU': 'yes'}, - 'resources': { - 'gpu': ['--gres=gpu:{num_gpus_per_node}'], - 'datawarp': [ + ] + }, + { + 'name': 'testsys', + 'descr': 'Fake system for unit tests', + 'hostnames': ['testsys'], + 'prefix': '.rfm_testing', + 'resourcesdir': '.rfm_testing/resources', + 'modules': ['foo/1.0'], + 'variables': [['FOO_CMD', 'foobar']], + 'partitions': [ + { + 'name': 'login', + 'scheduler': 'local', + 'launcher': 'local', + 'environs': ['PrgEnv-cray', 'PrgEnv-gnu', 'builtin-gcc'], + 'descr': 'Login nodes' + }, + { + 'name': 'gpu', + 'descr': 'GPU partition', + 'scheduler': 'slurm', + 'launcher': 'srun', + 'modules': ['foogpu'], + 'variables': [['FOO_GPU', 'yes']], + 'resources': [ + { + 'name': 'gpu', + 'options': ['--gres=gpu:{num_gpus_per_node}'], + }, + { + 'name': 'datawarp', + 'options': [ '#DW jobdw capacity={capacity}', '#DW stage_in source={stagein_src}' ] - }, - 'access': [], - 'environs': ['PrgEnv-gnu', 'builtin-gcc'], - 'descr': 'GPU partition', - } - } - }, - 'sys0': { - # System used for dependency checking - 'descr': 'System for checking test dependencies', - 'hostnames': [r'sys\d+'], - 'partitions': { - 'p0': { - 'scheduler': 'local', - 'environs': ['e0', 'e1'], - }, - 'p1': { - 'scheduler': 'local', - 'environs': ['e0', 'e1'], - } + } + ], + 'environs': ['PrgEnv-gnu', 'builtin-gcc'], + 'max_jobs': 10 } - } + ] }, - 'environments': { - 'testsys:login': { - 'PrgEnv-gnu': { - 'modules': ['PrgEnv-gnu'], - 'cc': 'gcc', - 'cxx': 'g++', - 'ftn': 'gfortran', - }, - }, - '*': { - 'PrgEnv-gnu': { - 'modules': ['PrgEnv-gnu'], - }, - 'PrgEnv-cray': { - 'modules': ['PrgEnv-cray'], + { + 'name': 'sys0', + 'descr': 'System for testing check dependencies', + 'hostnames': [r'sys\d+'], + 'partitions': [ + { + 'name': 'p0', + 'scheduler': 'local', + 'launcher': 'local', + 'environs': ['e0', 'e1'] }, - 'builtin': { - 'cc': 'cc', - 'cxx': '', - 'ftn': '', - }, - 'builtin-gcc': { - 'cc': 'gcc', - 'cxx': 'g++', - 'ftn': 'gfortran', - }, - 'e0': { - 'modules': ['m0'], - }, - 'e1': { - 'modules': ['m1'], + { + 'name': 'p1', + 'scheduler': 'local', + 'launcher': 'local', + 'environs': ['e0', 'e1'] + } + + ] + } + ], + 'environments': [ + { + 'name': 'PrgEnv-gnu', + 'modules': ['PrgEnv-gnu'], + }, + { + 'name': 'PrgEnv-gnu', + 'modules': ['PrgEnv-gnu'], + 'cc': 'gcc', + 'cxx': 'g++', + 'ftn': 'gfortran', + 'target_systems': ['testsys:login'] + }, + { + 'name': 'PrgEnv-cray', + 'modules': ['PrgEnv-cray'], + }, + { + 'name': 'builtin', + 'cc': 'cc', + 'cxx': '', + 'ftn': '' + }, + { + 'name': 'builtin-gcc', + 'cc': 'gcc', + 'cxx': 'g++', + 'ftn': 'gfortran' + }, + { + 'name': 'e0', + 'modules': ['m0'] + }, + { + 'name': 'e1', + 'modules': ['m1'] + }, + { + 'name': 'irrelevant', + 'target_systems': ['foo'] + } + ], + 'modes': [ + { + 'name': 'unittest', + 'options': [ + '-c', 'unittests/resources/checks/hellocheck.py', + '-p', 'builtin-gcc', + '--force-local' + ] + } + ], + 'logging': [ + { + 'level': 'debug', + 'handlers': [ + { + 'type': 'file', + 'name': '.rfm_unittest.log', + 'level': 'debug', + 'format': ( + '[%(check_job_completion_time)s] %(levelname)s: ' + '%(check_name)s: %(message)s' + ), + 'datefmt': '%FT%T', + 'append': False, }, - } + { + 'type': 'stream', + 'name': 'stdout', + 'level': 'info', + 'format': '%(message)s' + } + ], + 'handlers_perflog': [ + { + 'type': 'filelog', + 'prefix': '%(check_system)s/%(check_partition)s', + 'level': 'info', + 'format': ( + '%(check_job_completion_time)s|reframe %(version)s|' + '%(check_info)s|jobid=%(check_jobid)s|' + '%(check_perf_var)s=%(check_perf_value)s|' + 'ref=%(check_perf_ref)s ' + '(l=%(check_perf_lower_thres)s, ' + 'u=%(check_perf_upper_thres)s)|' + '%(check_perf_unit)s' + ), + 'append': True + } + ] + } + ], + 'general': [ + { + 'check_search_path': ['a:b'], + 'target_systems': ['testsys:login'] }, - 'modes': { - '*': { - 'unittest': [ - '-c', 'unittests/resources/checks/hellocheck.py', - '-p', 'builtin-gcc', - '--force-local' - ] - } + { + 'check_search_path': ['c:d'], + 'target_systems': ['testsys'] } - } - - logging_config = { - 'level': 'DEBUG', - 'handlers': [ - { - 'type': 'file', - 'name': '.rfm_unittest.log', - 'level': 'DEBUG', - 'format': ('[%(asctime)s] %(levelname)s: ' - '%(check_name)s: %(message)s'), - 'datefmt': '%FT%T', - 'append': False, - }, - { - 'type': 'stream', - 'name': 'stdout', - 'level': 'INFO', - 'format': '%(message)s' - }, - ] - } - - perf_logging_config = { - 'level': 'DEBUG', - 'handlers': [ - { - 'type': 'filelog', - 'prefix': '%(check_system)s/%(check_partition)s', - 'level': 'INFO', - 'format': ( - '%(check_job_completion_time)s|reframe %(version)s|' - '%(check_info)s|jobid=%(check_jobid)s|' - '%(check_perf_var)s=%(check_perf_value)s|' - 'ref=%(check_perf_ref)s ' - '(l=%(check_perf_lower_thres)s, ' - 'u=%(check_perf_upper_thres)s)|' - '%(check_perf_unit)s' - ), - 'datefmt': '%FT%T%:z', - 'append': True - } - ] - } - - -settings = ReframeSettings() + ] +} diff --git a/unittests/resources/settings_old_syntax.py b/unittests/resources/settings_old_syntax.py new file mode 100644 index 0000000000..057e3fa588 --- /dev/null +++ b/unittests/resources/settings_old_syntax.py @@ -0,0 +1,174 @@ +# Copyright 2016-2020 Swiss National Supercomputing Centre (CSCS/ETH Zurich) +# ReFrame Project Developers. See the top-level LICENSE file for details. +# +# SPDX-License-Identifier: BSD-3-Clause + +# +# ReFrame settings for use in the unit tests (old syntax) +# + + +class ReframeSettings: + job_poll_intervals = [1, 2, 3] + job_submit_timeout = 60 + checks_path = ['checks/'] + checks_path_recurse = True + site_configuration = { + 'systems': { + # Generic system configuration that allows to run ReFrame locally + # on any system. + 'generic': { + 'descr': 'Generic example system', + 'hostnames': ['localhost'], + 'partitions': { + 'login': { + 'scheduler': 'local', + 'modules': [], + 'access': [], + 'environs': ['builtin-gcc'], + 'descr': 'Login nodes' + }, + } + }, + 'testsys': { + # A fake system simulating a possible cluster configuration, in + # order to test different aspects of the framework. + 'descr': 'Fake system for unit tests', + 'hostnames': ['testsys'], + 'prefix': '.rfm_testing', + 'resourcesdir': '.rfm_testing/resources', + 'perflogdir': '.rfm_testing/perflogs', + 'modules': ['foo/1.0'], + 'variables': {'FOO_CMD': 'foobar'}, + 'partitions': { + 'login': { + 'scheduler': 'local', + 'resources': {}, + 'environs': ['PrgEnv-cray', + 'PrgEnv-gnu', + 'builtin-gcc'], + 'descr': 'Login nodes' + }, + 'gpu': { + 'scheduler': 'nativeslurm', + 'modules': ['foogpu'], + 'variables': {'FOO_GPU': 'yes'}, + 'resources': { + 'gpu': ['--gres=gpu:{num_gpus_per_node}'], + 'datawarp': [ + '#DW jobdw capacity={capacity}', + '#DW stage_in source={stagein_src}' + ] + }, + 'access': [], + 'environs': ['PrgEnv-gnu', 'builtin-gcc'], + 'descr': 'GPU partition', + } + } + }, + 'sys0': { + # System used for dependency checking + 'descr': 'System for checking test dependencies', + 'hostnames': [r'sys\d+'], + 'partitions': { + 'p0': { + 'scheduler': 'local', + 'environs': ['e0', 'e1'], + }, + 'p1': { + 'scheduler': 'local', + 'environs': ['e0', 'e1'], + } + } + } + }, + 'environments': { + 'testsys:login': { + 'PrgEnv-gnu': { + 'modules': ['PrgEnv-gnu'], + 'cc': 'gcc', + 'cxx': 'g++', + 'ftn': 'gfortran', + }, + }, + '*': { + 'PrgEnv-gnu': { + 'modules': ['PrgEnv-gnu'], + }, + 'PrgEnv-cray': { + 'modules': ['PrgEnv-cray'], + }, + 'builtin': { + 'cc': 'cc', + 'cxx': '', + 'ftn': '', + }, + 'builtin-gcc': { + 'cc': 'gcc', + 'cxx': 'g++', + 'ftn': 'gfortran', + }, + 'e0': { + 'modules': ['m0'], + }, + 'e1': { + 'modules': ['m1'], + }, + } + }, + 'modes': { + '*': { + 'unittest': [ + '-c', 'unittests/resources/checks/hellocheck.py', + '-p', 'builtin-gcc', + '--force-local' + ] + } + } + } + + logging_config = { + 'level': 'DEBUG', + 'handlers': [ + { + 'type': 'file', + 'name': '.rfm_unittest.log', + 'level': 'DEBUG', + 'format': ('[%(asctime)s] %(levelname)s: ' + '%(check_name)s: %(message)s'), + 'datefmt': '%FT%T', + 'append': False, + }, + { + 'type': 'stream', + 'name': 'stdout', + 'level': 'INFO', + 'format': '%(message)s' + }, + ] + } + + perf_logging_config = { + 'level': 'DEBUG', + 'handlers': [ + { + 'type': 'filelog', + 'prefix': '%(check_system)s/%(check_partition)s', + 'level': 'INFO', + 'format': ( + '%(check_job_completion_time)s|reframe %(version)s|' + '%(check_info)s|jobid=%(check_jobid)s|' + '%(check_perf_var)s=%(check_perf_value)s|' + 'ref=%(check_perf_ref)s ' + '(l=%(check_perf_lower_thres)s, ' + 'u=%(check_perf_upper_thres)s)|' + '%(check_perf_unit)s' + ), + 'datefmt': '%FT%T%:z', + 'append': True + } + ] + } + + +settings = ReframeSettings() diff --git a/unittests/test_argparser.py b/unittests/test_argparser.py index 2de1e83ccc..ecfec94ba8 100644 --- a/unittests/test_argparser.py +++ b/unittests/test_argparser.py @@ -4,61 +4,174 @@ # SPDX-License-Identifier: BSD-3-Clause import pytest -import unittest +import reframe.core.runtime as rt +import unittests.fixtures as fixtures from reframe.frontend.argparse import ArgumentParser -class TestArgumentParser(unittest.TestCase): - def setUp(self): - self.parser = ArgumentParser() - self.foo_options = self.parser.add_argument_group('foo options') - self.bar_options = self.parser.add_argument_group('bar options') - self.foo_options.add_argument('-f', '--foo', dest='foo', - action='store', default='FOO') - self.foo_options.add_argument('--foolist', dest='foolist', - action='append', default=[]) - self.foo_options.add_argument('--foobar', action='store_true') - self.foo_options.add_argument('--unfoo', action='store_false') - - self.bar_options.add_argument('-b', '--bar', dest='bar', - action='store', default='BAR') - self.bar_options.add_argument('--barlist', dest='barlist', - action='append', default=[]) - self.foo_options.add_argument('--barfoo', action='store_true') - - def test_arguments(self): - with pytest.raises(ValueError): - self.foo_options.add_argument(action='store', default='FOO') - - self.foo_options.add_argument('--foo-bar', action='store_true') - self.foo_options.add_argument('--alist', action='append', default=[]) - options = self.parser.parse_args(['--foobar', '--foo-bar']) - assert options.foobar - assert options.foo_bar - - def test_parsing(self): - options = self.parser.parse_args( - '--foo name --foolist gag --barfoo --unfoo'.split() +@pytest.fixture +def argparser(): + with rt.temp_runtime(fixtures.TEST_CONFIG_FILE): + return ArgumentParser() + + +@pytest.fixture +def foo_options(argparser): + opt_group = argparser.add_argument_group('foo options') + opt_group.add_argument('-f', '--foo', dest='foo', + action='store', default='FOO') + opt_group.add_argument('--foolist', dest='foolist', + action='append', default=[]) + opt_group.add_argument('--foobar', action='store_true') + opt_group.add_argument('--unfoo', action='store_false') + opt_group.add_argument('--barfoo', action='store_true') + return opt_group + + +@pytest.fixture +def bar_options(argparser): + opt_group = argparser.add_argument_group('bar options') + opt_group.add_argument('-b', '--bar', dest='bar', + action='store', default='BAR') + opt_group.add_argument('--barlist', dest='barlist', + action='append', default=[]) + return opt_group + + +def test_arguments(argparser, foo_options): + with pytest.raises(ValueError): + foo_options.add_argument(action='store', default='FOO') + + foo_options.add_argument('--foo-bar', action='store_true') + foo_options.add_argument('--alist', action='append', default=[]) + options = argparser.parse_args(['--foobar', '--foo-bar']) + assert options.foobar + assert options.foo_bar + + +def test_parsing(argparser, foo_options, bar_options): + options = argparser.parse_args( + '--foo name --foolist gag --barfoo --unfoo'.split() + ) + assert 'name' == options.foo + assert ['gag'] == options.foolist + assert options.barfoo + assert not options.unfoo + + # Check the defaults now + assert not options.foobar + assert 'BAR' == options.bar + assert [] == options.barlist + + # Reparse based on the already loaded options + options = argparser.parse_args( + '--bar beer --foolist any'.split(), options + ) + assert 'name' == options.foo + assert ['any'] == options.foolist + assert not options.foobar + assert not options.unfoo + assert 'beer' == options.bar + assert [] == options.barlist + assert options.barfoo + + +@pytest.fixture +def extended_parser(): + parser = ArgumentParser() + foo_options = parser.add_argument_group('Foo options') + bar_options = parser.add_argument_group('Bar options') + parser.add_argument( + '-R', '--recursive', action='store_true', + envvar='RFM_RECURSIVE', configvar='general/check_search_recursive' + ) + parser.add_argument( + '--non-default-craype', action='store_true', + envvar='RFM_NON_DEFAULT_CRAYPE', configvar='general/non_default_craype' + ) + parser.add_argument( + '--prefix', action='store', configvar='systems/prefix' + ) + parser.add_argument('--version', action='version', version='1.0') + parser.add_argument( + dest='keep_stage_files', action='store_true', + envvar='RFM_KEEP_STAGE_FILES', configvar='general/keep_stage_files' + ) + foo_options.add_argument( + '--timestamp', action='store', + envvar='RFM_TIMESTAMP_DIRS', configvar='general/timestamp_dirs' + ) + foo_options.add_argument( + '-C', '--config-file', action='store', envvar='RFM_CONFIG_FILE' + ) + foo_options.add_argument( + '--check-path', action='append', envvar='RFM_CHECK_SEARCH_PATH :' + ) + foo_options.add_argument( + '--stagedir', action='store', configvar='systems/stagedir', + default='/foo' + ) + bar_options.add_argument( + '--module', action='append', envvar='RFM_MODULES_PRELOAD' + ) + bar_options.add_argument( + '--nocolor', action='store_false', dest='colorize', + envvar='RFM_COLORIZE', configvar='general/colorize' + ) + return parser + + +def test_option_precedence(extended_parser): + with rt.temp_environment(variables={ + 'RFM_TIMESTAMP': '%F', + 'RFM_NON_DEFAULT_CRAYPE': 'yes', + 'RFM_MODULES_PRELOAD': 'a,b,c', + 'RFM_CHECK_SEARCH_PATH': 'x:y:z' + + }): + options = extended_parser.parse_args( + ['--timestamp=%FT%T', '--nocolor'] ) - assert 'name' == options.foo - assert ['gag'] == options.foolist - assert options.barfoo - assert not options.unfoo - - # Check the defaults now - assert not options.foobar - assert 'BAR' == options.bar - assert [] == options.barlist - - # Reparse based on the already loaded options - options = self.parser.parse_args( - '--bar beer --foolist any'.split(), options + assert options.recursive is None + assert options.timestamp == '%FT%T' + assert options.non_default_craype is True + assert options.config_file is None + assert options.prefix is None + assert options.stagedir == '/foo' + assert options.module == ['a', 'b', 'c'] + assert options.check_path == ['x', 'y', 'z'] + assert options.colorize is False + + +def test_option_with_config(extended_parser): + with rt.temp_environment(variables={ + 'RFM_TIMESTAMP': '%F', + 'RFM_NON_DEFAULT_CRAYPE': 'yes', + 'RFM_MODULES_PRELOAD': 'a,b,c', + 'RFM_KEEP_STAGE_FILES': 'no' + }): + site_config = rt.runtime().site_config + options = extended_parser.parse_args( + ['--timestamp=%FT%T', '--nocolor'] ) - assert 'name' == options.foo - assert ['any'] == options.foolist - assert not options.foobar - assert not options.unfoo - assert 'beer' == options.bar - assert [] == options.barlist - assert options.barfoo + options.update_config(site_config) + assert site_config.get('general/0/check_search_recursive') is True + assert site_config.get('general/0/timestamp_dirs') == '%FT%T' + assert site_config.get('general/0/non_default_craype') is True + assert site_config.get('systems/0/prefix') == '.' + assert site_config.get('general/0/colorize') is False + assert site_config.get('general/0/keep_stage_files') is False + + # Defaults specified in parser override those in configuration file + assert site_config.get('systems/0/stagedir') == '/foo' + + +def test_option_envvar_conversion_error(extended_parser): + with rt.temp_environment(variables={ + 'RFM_NON_DEFAULT_CRAYPE': 'foo', + }): + site_config = rt.runtime().site_config + options = extended_parser.parse_args(['--nocolor']) + errors = options.update_config(site_config) + assert len(errors) == 1 diff --git a/unittests/test_check_filters.py b/unittests/test_check_filters.py index de3c59e4d8..0eb34ea411 100644 --- a/unittests/test_check_filters.py +++ b/unittests/test_check_filters.py @@ -74,11 +74,11 @@ def test_have_prgenv(self): assert 1 == self.count_checks(filters.have_prgenv('env4')) assert 3 == self.count_checks(filters.have_prgenv('env1|env3')) - @rt.switch_runtime(fixtures.TEST_SITE_CONFIG, 'testsys') + @rt.switch_runtime(fixtures.TEST_CONFIG_FILE, 'testsys') def test_partition(self): - p = rt.runtime().system.partition('gpu') + p = fixtures.partition_by_name('gpu') assert 2 == self.count_checks(filters.have_partition([p])) - p = rt.runtime().system.partition('login') + p = fixtures.partition_by_name('login') assert 0 == self.count_checks(filters.have_partition([p])) def test_have_gpu_only(self): diff --git a/unittests/test_cli.py b/unittests/test_cli.py index e700e252e4..f7515ed803 100644 --- a/unittests/test_cli.py +++ b/unittests/test_cli.py @@ -3,15 +3,13 @@ # # SPDX-License-Identifier: BSD-3-Clause -import copy import itertools import os +import pathlib import pytest import re import sys -import tempfile -import unittest -from contextlib import redirect_stdout, redirect_stderr +from contextlib import redirect_stdout, redirect_stderr, suppress from io import StringIO import reframe.core.config as config @@ -19,7 +17,6 @@ import reframe.core.runtime as rt import reframe.utility.os_ext as os_ext import unittests.fixtures as fixtures -from reframe.core.exceptions import ReframeDeprecationWarning def run_command_inline(argv, funct, *args, **kwargs): @@ -49,459 +46,481 @@ def run_command_inline(argv, funct, *args, **kwargs): captured_stderr.getvalue()) -class TestFrontend(unittest.TestCase): - @property - def argv(self): - ret = ['./bin/reframe', '--prefix', self.prefix, '--nocolor'] - if self.mode: - ret += ['--mode', self.mode] - - if self.system: - ret += ['--system', self.system] - - if self.config_file: - ret += ['-C', self.config_file] - - ret += itertools.chain(*(['-c', c] for c in self.checkpath)) - ret += itertools.chain(*(['-p', e] for e in self.environs)) - - if self.local: - ret += ['--force-local'] - - if self.action == 'run': - ret += ['-r'] - elif self.action == 'list': - ret += ['-l'] - elif self.action == 'list_detailed': - ret += ['-L'] - elif self.action == 'help': - ret += ['-h'] - - if self.ignore_check_conflicts: - ret += ['--ignore-check-conflicts'] - - if self.perflogdir: - ret += ['--perflogdir', self.perflogdir] - - ret += self.more_options - return ret - - def setUp(self): - self.prefix = tempfile.mkdtemp(dir='unittests') - self.system = 'generic:login' - self.checkpath = ['unittests/resources/checks/hellocheck.py'] - self.environs = ['builtin-gcc'] - self.local = True - self.action = 'run' - self.more_options = [] - self.mode = None - self.config_file = 'unittests/resources/settings.py' - self.logfile = '.rfm_unittest.log' - self.ignore_check_conflicts = True - self.perflogdir = '.rfm-perflogs' - - def tearDown(self): - os_ext.rmtree(self.prefix) - os_ext.rmtree(self.perflogdir, ignore_errors=True) - os_ext.force_remove_file(self.logfile) - - def _run_reframe(self): +@pytest.fixture +def logfile(): + path = pathlib.PosixPath('.rfm_unittest.log') + yield path + with suppress(FileNotFoundError): + path.unlink() + + +@pytest.fixture +def perflogdir(tmp_path): + dirname = tmp_path / '.rfm-perflogs' + yield dirname + + +@pytest.fixture +def run_reframe(tmp_path, logfile, perflogdir): + def _run_reframe(system='generic:default', + checkpath=['unittests/resources/checks/hellocheck.py'], + environs=['builtin-gcc'], + local=True, + action='run', + more_options=None, + mode=None, + config_file='unittests/resources/settings.py', + logfile=str(logfile), + ignore_check_conflicts=True, + perflogdir=str(perflogdir)): import reframe.frontend.cli as cli - return run_command_inline(self.argv, cli.main) - - def _stage_exists(self, check_name, partitions, environs): - stagedir = os.path.join(self.prefix, 'stage', 'generic') - for p in partitions: - for e in environs: - path = os.path.join(stagedir, p, e, check_name) - if not os.path.exists(path): - return False - - return True - - def _perflog_exists(self, check_name): - logfile = '.rfm-perflogs/generic/login/%s.log' % check_name - return os.path.exists(logfile) - - def assert_log_file_is_saved(self): - outputdir = os.path.join(self.prefix, 'output') - assert os.path.exists(self.logfile) - assert os.path.exists( - os.path.join(outputdir, os.path.basename(self.logfile))) - - def test_default_settings(self): - # Simply make sure that a default settings file exists - try: - import reframe.settings as settings - except ImportError: - pytest.fail('default settings file could not be found') - - def test_check_success(self): - self.more_options = ['--save-log-files'] - returncode, stdout, _ = self._run_reframe() - assert 'PASSED' in stdout - assert 'FAILED' not in stdout - assert 0 == returncode - self.assert_log_file_is_saved() - - @fixtures.switch_to_user_runtime - def test_check_submit_success(self): - # This test will run on the auto-detected system - partition = fixtures.partition_with_scheduler() - if not partition: - pytest.skip('job submission not supported') - - self.config_file = fixtures.USER_CONFIG_FILE - self.local = False - self.system = partition.fullname + argv = ['./bin/reframe', '--prefix', str(tmp_path), '--nocolor'] + if mode: + argv += ['--mode', mode] + + if system: + argv += ['--system', system] + + if config_file: + argv += ['-C', config_file] + + argv += itertools.chain(*(['-c', c] for c in checkpath)) + argv += itertools.chain(*(['-p', e] for e in environs)) + if local: + argv += ['--force-local'] + + if action == 'run': + argv += ['-r'] + elif action == 'list': + argv += ['-l'] + elif action == 'list_detailed': + argv += ['-L'] + elif action == 'help': + argv += ['-h'] + + if ignore_check_conflicts: + argv += ['--ignore-check-conflicts'] + + if perflogdir: + argv += ['--perflogdir', perflogdir] + + if more_options: + argv += more_options + + return run_command_inline(argv, cli.main) + + return _run_reframe + + +@pytest.fixture +def temp_runtime(tmp_path): + def _temp_runtime(site_config, system=None, options={}): + options.update({'systems/prefix': tmp_path}) + with rt.temp_runtime(site_config, system, options): + yield rt.runtime + + yield _temp_runtime + + +@pytest.fixture +def user_exec_ctx(temp_runtime): + if fixtures.USER_CONFIG_FILE is None: + pytest.skip('no user configuration file supplied') + + yield from temp_runtime(fixtures.USER_CONFIG_FILE, fixtures.USER_SYSTEM) + + +@pytest.fixture +def remote_exec_ctx(user_exec_ctx): + partition = fixtures.partition_by_scheduler() + if not partition: + pytest.skip('job submission not supported') + + return partition, partition.environs[0] + + +def test_check_success(run_reframe, tmp_path, logfile): + returncode, stdout, _ = run_reframe(more_options=['--save-log-files']) + assert 'PASSED' in stdout + assert 'FAILED' not in stdout + assert returncode == 0 + os.path.exists(tmp_path / 'output' / logfile) + + +def test_check_submit_success(run_reframe, remote_exec_ctx): + # This test will run on the auto-detected system + partition, environ = remote_exec_ctx + returncode, stdout, _ = run_reframe( + config_file=fixtures.USER_CONFIG_FILE, + local=False, + system=partition.fullname, # Pick up the programming environment of the partition # Prepend ^ and append $ so as to much exactly the given name - self.environs = ['^' + partition.environs[0].name + '$'] + environs=[f'^{environ.name}$'] + ) + + assert 'FAILED' not in stdout + assert 'PASSED' in stdout + + # Assert that we have run only one test case + assert 'Ran 1 test case(s)' in stdout + assert 0 == returncode - returncode, stdout, _ = self._run_reframe() - assert 'FAILED' not in stdout - assert 'PASSED' in stdout - # Assert that we have run only one test case - assert 'Ran 1 test case(s)' in stdout - assert 0 == returncode +def test_check_failure(run_reframe): + returncode, stdout, _ = run_reframe( + checkpath=['unittests/resources/checks/frontend_checks.py'], + more_options=['-t', 'BadSetupCheck'] + ) + assert 'FAILED' in stdout + assert returncode != 0 - def test_check_failure(self): - self.checkpath = ['unittests/resources/checks/frontend_checks.py'] - self.more_options = ['-t', 'BadSetupCheck'] - returncode, stdout, _ = self._run_reframe() - assert 'FAILED' in stdout - assert returncode != 0 +def test_check_setup_failure(run_reframe): + returncode, stdout, stderr = run_reframe( + checkpath=['unittests/resources/checks/frontend_checks.py'], + more_options=['-t', 'BadSetupCheckEarly'], + local=False, - def test_check_setup_failure(self): - self.checkpath = ['unittests/resources/checks/frontend_checks.py'] - self.more_options = ['-t', 'BadSetupCheckEarly'] - self.local = False + ) + assert 'Traceback' not in stdout + assert 'Traceback' not in stderr + assert 'FAILED' in stdout + assert returncode != 0 - returncode, stdout, stderr = self._run_reframe() - assert 'Traceback' not in stdout - assert 'Traceback' not in stderr - assert 'FAILED' in stdout - assert returncode != 0 - def test_check_kbd_interrupt(self): - self.checkpath = [ +def test_check_kbd_interrupt(run_reframe): + returncode, stdout, stderr = run_reframe( + checkpath=[ 'unittests/resources/checks_unlisted/kbd_interrupt.py' - ] - self.more_options = ['-t', 'KeyboardInterruptCheck'] - self.local = False - - returncode, stdout, stderr = self._run_reframe() - assert 'Traceback' not in stdout - assert 'Traceback' not in stderr - assert 'FAILED' in stdout - assert returncode != 0 - - def test_check_sanity_failure(self): - self.checkpath = ['unittests/resources/checks/frontend_checks.py'] - self.more_options = ['-t', 'SanityFailureCheck'] - - returncode, stdout, stderr = self._run_reframe() - assert 'FAILED' in stdout - - # This is a normal failure, it should not raise any exception - assert 'Traceback' not in stdout - assert 'Traceback' not in stderr - assert returncode != 0 - assert self._stage_exists('SanityFailureCheck', ['login'], - self.environs) - - def test_performance_check_failure(self): - self.checkpath = ['unittests/resources/checks/frontend_checks.py'] - self.more_options = ['-t', 'PerformanceFailureCheck'] - returncode, stdout, stderr = self._run_reframe() - - assert 'FAILED' in stdout - - # This is a normal failure, it should not raise any exception - assert 'Traceback' not in stdout - assert 'Traceback' not in stderr - assert 0 != returncode - assert self._stage_exists('PerformanceFailureCheck', ['login'], - self.environs) - assert self._perflog_exists('PerformanceFailureCheck') - - def test_failure_stats(self): - self.checkpath = ['unittests/resources/checks/frontend_checks.py'] - self.more_options = ['-t', 'SanityFailureCheck', '--failure-stats'] - returncode, stdout, stderr = self._run_reframe() - - assert r'FAILURE STATISTICS' in stdout - assert r'sanity 1 [SanityFailureCheck' in stdout - assert 'Traceback' not in stdout - assert 'Traceback' not in stderr - assert returncode != 0 - - def test_performance_report(self): - self.checkpath = ['unittests/resources/checks/frontend_checks.py'] - self.more_options = ['-t', 'PerformanceFailureCheck', - '--performance-report'] - returncode, stdout, stderr = self._run_reframe() - - assert r'PERFORMANCE REPORT' in stdout - assert r'perf: 10 Gflop/s' in stdout - - def test_skip_system_check_option(self): - self.checkpath = ['unittests/resources/checks/frontend_checks.py'] - self.more_options = ['--skip-system-check', '-t', 'NoSystemCheck'] - returncode, stdout, _ = self._run_reframe() - assert 'PASSED' in stdout - - def test_skip_prgenv_check_option(self): - self.checkpath = ['unittests/resources/checks/frontend_checks.py'] - self.more_options = ['--skip-prgenv-check', '-t', 'NoPrgEnvCheck'] - returncode, stdout, _ = self._run_reframe() - assert 'PASSED' in stdout - assert 0 == returncode - - def test_sanity_of_checks(self): - # This test will effectively load all the tests in the checks path and - # will force a syntactic and runtime check at least for the constructor - # of the checks - self.action = 'list' - self.more_options = ['--save-log-files'] - self.checkpath = [] - returncode, *_ = self._run_reframe() - - assert 0 == returncode - self.assert_log_file_is_saved() - - def test_unknown_system(self): - self.action = 'list' - self.system = 'foo' - self.checkpath = [] - returncode, stdout, stderr = self._run_reframe() - assert 'Traceback' not in stdout - assert 'Traceback' not in stderr - assert 1 == returncode - - def test_sanity_of_optconfig(self): - # Test the sanity of the command line options configuration - self.action = 'help' - self.checkpath = [] - returncode, *_ = self._run_reframe() - assert 0 == returncode - - def test_checkpath_colon_separated(self): - self.action = 'list' - self.checkpath = ['unittests/resources/checks/hellocheck_make.py:' - 'unittests/resources/checks/hellocheck.py'] - returncode, stdout, _ = self._run_reframe() - num_checks = re.search( - r'Found (\d+) check', stdout, re.MULTILINE).group(1) - assert num_checks == '2' - - def test_checkpath_symlink(self): - self.action = 'list' - self.checkpath = ['unittests/resources/checks'] - self.more_options = ['-R'] - returncode, stdout, _ = self._run_reframe() - num_checks_default = re.search( - r'Found (\d+) check', stdout, re.MULTILINE).group(1) - - with tempfile.TemporaryDirectory(dir='unittests') as tmp: - checks_link = os.path.join(tmp, 'checks_symlink') - os.symlink(os.path.abspath('unittests/resources/checks'), - os.path.abspath(checks_link)) - self.checkpath = ['unittests/resources/checks', checks_link] - returncode, stdout, _ = self._run_reframe() - num_checks_in_checkdir = re.search( - r'Found (\d+) check', stdout, re.MULTILINE).group(1) - - assert num_checks_in_checkdir == num_checks_default - - def test_checkpath_recursion(self): - self.action = 'list' - self.checkpath = [] - returncode, stdout, _ = self._run_reframe() - num_checks_default = re.search( - r'Found (\d+) check', stdout, re.MULTILINE).group(1) - - self.checkpath = ['checks/'] - self.more_options = ['-R'] - returncode, stdout, _ = self._run_reframe() - num_checks_in_checkdir = re.search( - r'Found (\d+) check', stdout, re.MULTILINE).group(1) - assert num_checks_in_checkdir == num_checks_default - - self.more_options = [] - returncode, stdout, stderr = self._run_reframe() - num_checks_in_checkdir = re.search( - r'Found (\d+) check', stdout, re.MULTILINE).group(1) - assert '0' == num_checks_in_checkdir - - def test_same_output_stage_dir(self): - output_dir = os.path.join(self.prefix, 'foo') - self.more_options = ['-o', output_dir, '-s', output_dir] - returncode, *_ = self._run_reframe() - assert 1 == returncode - - # retry with --keep-stage-files - self.more_options.append('--keep-stage-files') - returncode, *_ = self._run_reframe() - assert 0 == returncode - assert os.path.exists(output_dir) - - def test_execution_modes(self): - self.checkpath = [] - self.environs = [] - self.local = False - self.mode = 'unittest' - - returncode, stdout, stderr = self._run_reframe() - assert 'Traceback' not in stdout - assert 'Traceback' not in stderr - assert 'FAILED' not in stdout - assert 'PASSED' in stdout - assert 'Ran 1 test case' in stdout - - def test_no_ignore_check_conflicts(self): - self.checkpath = ['unittests/resources/checks'] - self.more_options = ['-R'] - self.ignore_check_conflicts = False - self.action = 'list' - returncode, *_ = self._run_reframe() - assert 0 != returncode - - def test_timestamp_option(self): - from datetime import datetime - - self.checkpath = ['unittests/resources/checks'] - self.more_options = ['-R'] - self.ignore_check_conflicts = False - self.action = 'list' - self.more_options = ['--timestamp=xxx_%F'] - timefmt = datetime.now().strftime('xxx_%F') - returncode, stdout, _ = self._run_reframe() - assert 0 != returncode - assert timefmt in stdout - - def test_list_empty_prgenvs_check_and_options(self): - self.checkpath = ['unittests/resources/checks/frontend_checks.py'] - self.action = 'list' - self.environs = [] - self.more_options = ['-n', 'NoPrgEnvCheck'] - returncode, stdout, _ = self._run_reframe() - assert 'Found 0 check(s)' in stdout - assert 0 == returncode - - def test_list_check_with_empty_prgenvs(self): - self.checkpath = ['unittests/resources/checks/frontend_checks.py'] - self.action = 'list' - self.environs = ['foo'] - self.more_options = ['-n', 'NoPrgEnvCheck'] - returncode, stdout, _ = self._run_reframe() - assert 'Found 0 check(s)' in stdout - assert 0 == returncode - - def test_list_empty_prgenvs_in_check_and_options(self): - self.checkpath = ['unittests/resources/checks/frontend_checks.py'] - self.action = 'list' - self.environs = [] - self.more_options = ['-n', 'NoPrgEnvCheck'] - returncode, stdout, _ = self._run_reframe() - assert 'Found 0 check(s)' in stdout - assert 0 == returncode - - def test_list_with_details(self): - self.checkpath = ['unittests/resources/checks/frontend_checks.py'] - self.action = 'list_detailed' - returncode, stdout, stderr = self._run_reframe() - assert 'Traceback' not in stdout - assert 'Traceback' not in stderr - assert 0 == returncode - - def test_show_config(self): - # Just make sure that this option does not make the frontend crash - self.more_options = ['--show-config'] - self.system = 'testsys' - returncode, stdout, stderr = self._run_reframe() - assert 'Traceback' not in stdout - assert 'Traceback' not in stderr - assert 0 == returncode - - def test_show_env_config(self): - # Just make sure that this option does not make the frontend crash - self.more_options = ['--show-config-env', 'PrgEnv-gnu'] - self.system = 'testsys' - returncode, stdout, stderr = self._run_reframe() - assert 'Traceback' not in stdout - assert 'Traceback' not in stderr - assert 0 == returncode - - def test_show_env_config_unknown_env(self): - # Just make sure that this option does not make the frontend crash - self.more_options = ['--show-config-env', 'foobar'] - self.system = 'testsys' - returncode, stdout, stderr = self._run_reframe() - assert 'Traceback' not in stdout - assert 'Traceback' not in stderr - assert 1 == returncode - - def test_no_deprecation_warnings(self): - self.action = 'run' - self.checkpath = [ - 'unittests/resources/checks_unlisted/deprecated_test.py' - ] - with pytest.warns(ReframeDeprecationWarning): - returncode, stdout, stderr = self._run_reframe() - - self.more_options = ['--no-deprecation-warnings'] - - # We get the list of captured `warnings.WarningMessage` objects - with pytest.warns(None) as warnings: - self._run_reframe() - - # The `message` field contains the actual exception object - assert not any(isinstance(w.message, ReframeDeprecationWarning) - for w in warnings) - - def test_verbosity(self): - self.more_options = ['-vvvvv'] - self.system = 'testsys' - self.action = 'list' - returncode, stdout, stderr = self._run_reframe() - assert '' != stdout - assert 'Traceback' not in stdout - assert 'Traceback' not in stderr - assert 0 == returncode - - def test_verbosity_with_check(self): - self.more_options = ['-vvvvv'] - self.checkpath = ['unittests/resources/checks/hellocheck.py'] - returncode, stdout, stderr = self._run_reframe() - assert '' != stdout - assert '--- Logging error ---' not in stdout - assert 'Traceback' not in stdout - assert 'Traceback' not in stderr - assert 0 == returncode - - @fixtures.switch_to_user_runtime - def test_unload_module(self): - # This test is mostly for ensuring coverage. `_run_reframe()` restores - # the current environment, so it is not easy to verify that the modules - # are indeed unloaded. However, this functionality is tested elsewhere - # more exhaustively. - - ms = rt.runtime().modules_system - if ms.name == 'nomod': - pytest.skip('no modules system found') - - with rt.module_use('unittests/modules'): - ms.load_module('testmod_foo') - self.more_options = ['-u testmod_foo'] - self.action = 'list' - returncode, stdout, stderr = self._run_reframe() - ms.unload_module('testmod_foo') - - assert stdout != '' - assert 'Traceback' not in stdout - assert 'Traceback' not in stderr - assert returncode == 0 + ], + more_options=['-t', 'KeyboardInterruptCheck'], + local=False, + ) + assert 'Traceback' not in stdout + assert 'Traceback' not in stderr + assert 'FAILED' in stdout + assert returncode != 0 + + +def test_check_sanity_failure(run_reframe, tmp_path): + returncode, stdout, stderr = run_reframe( + checkpath=['unittests/resources/checks/frontend_checks.py'], + more_options=['-t', 'SanityFailureCheck'] + ) + assert 'FAILED' in stdout + + # This is a normal failure, it should not raise any exception + assert 'Traceback' not in stdout + assert 'Traceback' not in stderr + assert returncode != 0 + assert os.path.exists( + tmp_path / 'stage' / 'generic' / 'default' / + 'builtin-gcc' / 'SanityFailureCheck' + ) + + +def test_checkpath_symlink(run_reframe, tmp_path): + # FIXME: This should move to test_loader.py + checks_symlink = tmp_path / 'checks_symlink' + os.symlink(os.path.abspath('unittests/resources/checks'), + checks_symlink) + + returncode, stdout, _ = run_reframe( + action='list', + more_options=['-R'], + checkpath=['unittests/resources/checks', str(checks_symlink)] + ) + num_checks_default = re.search( + r'Found (\d+) check', stdout, re.MULTILINE).group(1) + num_checks_in_checkdir = re.search( + r'Found (\d+) check', stdout, re.MULTILINE).group(1) + assert num_checks_in_checkdir == num_checks_default + + +def test_performance_check_failure(run_reframe, tmp_path, perflogdir): + returncode, stdout, stderr = run_reframe( + checkpath=['unittests/resources/checks/frontend_checks.py'], + more_options=['-t', 'PerformanceFailureCheck'] + ) + assert 'FAILED' in stdout + + # This is a normal failure, it should not raise any exception + assert 'Traceback' not in stdout + assert 'Traceback' not in stderr + assert returncode != 0 + assert os.path.exists( + tmp_path / 'stage' / 'generic' / 'default' / + 'builtin-gcc' / 'PerformanceFailureCheck' + ) + assert os.path.exists(perflogdir / 'generic' / + 'default' / 'PerformanceFailureCheck.log') + + +def test_performance_report(run_reframe): + returncode, stdout, stderr = run_reframe( + checkpath=['unittests/resources/checks/frontend_checks.py'], + more_options=['-t', 'PerformanceFailureCheck', '--performance-report'] + ) + assert r'PERFORMANCE REPORT' in stdout + assert r'perf: 10 Gflop/s' in stdout + + +def test_skip_system_check_option(run_reframe): + returncode, stdout, _ = run_reframe( + checkpath=['unittests/resources/checks/frontend_checks.py'], + more_options=['--skip-system-check', '-t', 'NoSystemCheck'] + ) + assert 'PASSED' in stdout + assert returncode == 0 + + +def test_skip_prgenv_check_option(run_reframe): + returncode, stdout, _ = run_reframe( + checkpath=['unittests/resources/checks/frontend_checks.py'], + more_options=['--skip-prgenv-check', '-t', 'NoPrgEnvCheck'] + ) + assert 'PASSED' in stdout + assert returncode == 0 + + +def test_sanity_of_checks(run_reframe, tmp_path, logfile): + # This test will effectively load all the tests in the checks path and + # will force a syntactic and runtime check at least for the constructor + # of the checks + returncode, *_ = run_reframe( + action='list', + more_options=['--save-log-files'], + checkpath=[] + ) + assert returncode == 0 + os.path.exists(tmp_path / 'output' / logfile) + + +def test_unknown_system(run_reframe): + returncode, stdout, stderr = run_reframe( + action='list', + system='foo', + checkpath=[] + ) + assert 'Traceback' not in stdout + assert 'Traceback' not in stderr + assert returncode == 1 + + +def test_sanity_of_optconfig(run_reframe): + # Test the sanity of the command line options configuration + returncode, *_ = run_reframe( + action='help', + checkpath=[] + ) + assert returncode == 0 + + +def test_checkpath_recursion(run_reframe): + _, stdout, _ = run_reframe(action='list', checkpath=[]) + num_checks_default = re.search(r'Found (\d+) check', stdout).group(1) + + _, stdout, _ = run_reframe(action='list', + checkpath=['checks/'], + more_options=['-R']) + num_checks_in_checkdir = re.search(r'Found (\d+) check', stdout).group(1) + assert num_checks_in_checkdir == num_checks_default + + _, stdout, _ = run_reframe(action='list', + checkpath=['checks/'], + more_options=[]) + num_checks_in_checkdir = re.search(r'Found (\d+) check', stdout).group(1) + assert num_checks_in_checkdir == '0' + + +def test_same_output_stage_dir(run_reframe, tmp_path): + output_dir = str(tmp_path / 'foo') + returncode, *_ = run_reframe( + more_options=['-o', output_dir, '-s', output_dir] + ) + assert returncode == 1 + + # Retry with --keep-stage-files + returncode, *_ = run_reframe( + more_options=['-o', output_dir, '-s', output_dir, '--keep-stage-files'] + ) + assert returncode == 0 + assert os.path.exists(output_dir) + + +def test_execution_modes(run_reframe): + returncode, stdout, stderr = run_reframe( + checkpath=[], + environs=[], + local=False, + mode='unittest' + ) + assert 'Traceback' not in stdout + assert 'Traceback' not in stderr + assert 'FAILED' not in stdout + assert 'PASSED' in stdout + assert 'Ran 1 test case' in stdout + + +def test_no_ignore_check_conflicts(run_reframe): + returncode, *_ = run_reframe( + checkpath=['unittests/resources/checks'], + more_options=['-R'], + ignore_check_conflicts=False, + action='list' + ) + assert returncode != 0 + + +def test_timestamp_option(run_reframe): + from datetime import datetime + + timefmt = datetime.now().strftime('xxx_%F') + returncode, stdout, _ = run_reframe( + checkpath=['unittests/resources/checks'], + ignore_check_conflicts=False, + action='list', + more_options=['-R', '--timestamp=xxx_%F'] + ) + assert returncode != 0 + assert timefmt in stdout + + +def test_list_empty_prgenvs_check_and_options(run_reframe): + returncode, stdout, _ = run_reframe( + checkpath=['unittests/resources/checks/frontend_checks.py'], + action='list', + environs=[], + more_options=['-n', 'NoPrgEnvCheck'], + ) + assert 'Found 0 check(s)' in stdout + assert returncode == 0 + + +def test_list_check_with_empty_prgenvs(run_reframe): + returncode, stdout, _ = run_reframe( + checkpath=['unittests/resources/checks/frontend_checks.py'], + action='list', + environs=['foo'], + more_options=['-n', 'NoPrgEnvCheck'] + ) + assert 'Found 0 check(s)' in stdout + assert returncode == 0 + + +def test_list_empty_prgenvs_in_check_and_options(run_reframe): + returncode, stdout, _ = run_reframe( + checkpath=['unittests/resources/checks/frontend_checks.py'], + action='list', + environs=[], + more_options=['-n', 'NoPrgEnvCheck'] + ) + assert 'Found 0 check(s)' in stdout + assert returncode == 0 + + +def test_list_with_details(run_reframe): + returncode, stdout, stderr = run_reframe( + checkpath=['unittests/resources/checks/frontend_checks.py'], + action='list_detailed' + ) + assert 'Traceback' not in stdout + assert 'Traceback' not in stderr + assert returncode == 0 + + +def test_show_config_all(run_reframe): + # Just make sure that this option does not make the frontend crash + returncode, stdout, stderr = run_reframe( + more_options=['--show-config'], + system='testsys' + ) + assert 'Traceback' not in stdout + assert 'Traceback' not in stderr + assert returncode == 0 + + +def test_show_config_param(run_reframe): + # Just make sure that this option does not make the frontend crash + returncode, stdout, stderr = run_reframe( + more_options=['--show-config=systems'], + system='testsys' + ) + assert 'Traceback' not in stdout + assert 'Traceback' not in stderr + assert returncode == 0 + + +def test_show_config_unknown_param(run_reframe): + # Just make sure that this option does not make the frontend crash + returncode, stdout, stderr = run_reframe( + more_options=['--show-config=foo'], + system='testsys' + ) + assert 'no such configuration parameter found' in stdout + assert 'Traceback' not in stdout + assert 'Traceback' not in stderr + assert returncode == 0 + + +def test_verbosity(run_reframe): + returncode, stdout, stderr = run_reframe( + more_options=['-vvvvv'], + system='testsys', + action='list' + ) + assert stdout != '' + assert 'Traceback' not in stdout + assert 'Traceback' not in stderr + assert returncode == 0 + + +def test_verbosity_with_check(run_reframe): + returncode, stdout, stderr = run_reframe( + more_options=['-vvvvv'], + system='testsys', + checkpath=['unittests/resources/checks/hellocheck.py'] + ) + assert '' != stdout + assert '--- Logging error ---' not in stdout + assert 'Traceback' not in stdout + assert 'Traceback' not in stderr + assert 0 == returncode + + +def test_unload_module(run_reframe, user_exec_ctx): + # This test is mostly for ensuring coverage. `run_reframe()` restores + # the current environment, so it is not easy to verify that the modules + # are indeed unloaded. However, this functionality is tested elsewhere + # more exhaustively. + + ms = rt.runtime().modules_system + if ms.name == 'nomod': + pytest.skip('no modules system found') + + with rt.module_use('unittests/modules'): + ms.load_module('testmod_foo') + returncode, stdout, stderr = run_reframe( + more_options=['-u testmod_foo'], + action='list' + ) + ms.unload_module('testmod_foo') + + assert stdout != '' + assert 'Traceback' not in stdout + assert 'Traceback' not in stderr + assert returncode == 0 + + +def test_failure_stats(run_reframe): + returncode, stdout, stderr = run_reframe( + checkpath=['unittests/resources/checks/frontend_checks.py'], + more_options=['-t', 'SanityFailureCheck', '--failure-stats'] + ) + assert r'FAILURE STATISTICS' in stdout + assert r'sanity 1 [SanityFailureCheck' in stdout + assert 'Traceback' not in stdout + assert 'Traceback' not in stderr + assert returncode != 0 diff --git a/unittests/test_config.py b/unittests/test_config.py index 8a86df044d..836b742b54 100644 --- a/unittests/test_config.py +++ b/unittests/test_config.py @@ -3,153 +3,334 @@ # # SPDX-License-Identifier: BSD-3-Clause -import copy +import json +import os import pytest -import unittest import reframe.core.config as config -import unittests.fixtures as fixtures -from reframe.core.exceptions import ConfigError - - -class TestSiteConfigurationFromDict(unittest.TestCase): - def setUp(self): - self.site_config = config.SiteConfiguration() - self.dict_config = copy.deepcopy(fixtures.TEST_SITE_CONFIG) - - def get_partition(self, system, name): - for p in system.partitions: - if p.name == name: - return p - - def test_load_success(self): - self.site_config.load_from_dict(self.dict_config) - assert len(self.site_config.systems) == 3 - - system = self.site_config.systems['testsys'] - assert len(system.partitions) == 2 - assert system.prefix == '.rfm_testing' - assert system.resourcesdir == '.rfm_testing/resources' - assert system.perflogdir == '.rfm_testing/perflogs' - assert system.preload_environ.modules == ['foo/1.0'] - assert system.preload_environ.variables == {'FOO_CMD': 'foobar'} - - part_login = self.get_partition(system, 'login') - part_gpu = self.get_partition(system, 'gpu') - assert part_login is not None - assert part_gpu is not None - assert part_login.fullname == 'testsys:login' - assert part_gpu.fullname == 'testsys:gpu' - assert len(part_login.environs) == 3 - assert len(part_gpu.environs) == 2 - - # Check local partition environment - assert part_gpu.local_env.modules == ['foogpu'] - assert part_gpu.local_env.variables == {'FOO_GPU': 'yes'} - - # Check that PrgEnv-gnu on login partition is resolved to the special - # version defined in the 'dom:login' section - env_login = part_login.environment('PrgEnv-gnu') - assert env_login.cc == 'gcc' - assert env_login.cxx == 'g++' - assert env_login.ftn == 'gfortran' - - # Check that the PrgEnv-gnu of the gpu partition is resolved to the - # default one - env_gpu = part_gpu.environment('PrgEnv-gnu') - assert env_gpu.cc == 'cc' - assert env_gpu.cxx == 'CC' - assert env_gpu.ftn == 'ftn' - - # Check resource instantiation - resource_spec = part_gpu.get_resource('gpu', num_gpus_per_node=16) - assert (resource_spec == ['--gres=gpu:16']) - - resources_spec = part_gpu.get_resource('datawarp', - capacity='100GB', - stagein_src='/foo') - assert (resources_spec == ['#DW jobdw capacity=100GB', - '#DW stage_in source=/foo']) - - def test_load_envconfig_with_unknown_args(self): - self.dict_config['environments']['*']['builtin-gcc'] = { - 'foo': 'bar', - } - self.site_config.load_from_dict(self.dict_config) - - def test_load_failure_empty_dict(self): - dict_config = {} - with pytest.raises(ValueError): - self.site_config.load_from_dict(dict_config) - - def test_load_failure_no_environments(self): - dict_config = {'systems': {}} - with pytest.raises(ValueError): - self.site_config.load_from_dict(dict_config) - - def test_load_failure_no_systems(self): - dict_config = {'environments': {}} - with pytest.raises(ValueError): - self.site_config.load_from_dict(dict_config) - - def test_load_failure_environments_no_scoped_dict(self): - self.dict_config['environments'] = { - 'testsys': 'PrgEnv-gnu' - } - with pytest.raises(TypeError): - self.site_config.load_from_dict(self.dict_config) - - def test_load_failure_partitions_nodict(self): - self.dict_config['systems']['testsys']['partitions'] = ['gpu'] - with pytest.raises(ConfigError): - self.site_config.load_from_dict(self.dict_config) - - def test_load_failure_systems_nodict(self): - self.dict_config['systems']['testsys'] = ['gpu'] - with pytest.raises(TypeError): - self.site_config.load_from_dict(self.dict_config) - - def test_load_failure_partitions_nodict(self): - self.dict_config['systems']['testsys']['partitions']['login'] = 'foo' - with pytest.raises(TypeError): - self.site_config.load_from_dict(self.dict_config) - - def test_load_failure_partconfig_nodict(self): - self.dict_config['systems']['testsys']['partitions']['login'] = 'foo' - with pytest.raises(TypeError): - self.site_config.load_from_dict(self.dict_config) - - def test_load_failure_unresolved_environment(self): - self.dict_config['environments'] = { - '*': { - 'PrgEnv-gnu': { - 'modules': ['PrgEnv-gnu'], - } - } - } - with pytest.raises(ConfigError, - match='could not find a definition for'): - self.site_config.load_from_dict(self.dict_config) - - def test_load_failure_envconfig_nodict(self): - self.dict_config['environments']['*']['PrgEnv-gnu'] = 'foo' - with pytest.raises(TypeError): - self.site_config.load_from_dict(self.dict_config) - - -class TestConfigLoading(unittest.TestCase): - def test_load_normal_config(self): - config.load_settings_from_file('unittests/resources/settings.py') - - def test_load_unknown_file(self): - with pytest.raises(ConfigError): - config.load_settings_from_file('foo') - - def test_load_no_settings(self): - with pytest.raises(ConfigError): - config.load_settings_from_file('unittests') - - def test_load_invalid_settings(self): - with pytest.raises(ConfigError): - config.load_settings_from_file( - 'unittests/resources/invalid_settings.py') +import reframe.utility.os_ext as os_ext +from reframe.core.exceptions import (ConfigError, ReframeDeprecationWarning) +from reframe.core.systems import System + + +def test_load_config_fallback(monkeypatch): + monkeypatch.setattr(config, '_find_config_file', lambda: None) + site_config = config.load_config() + assert site_config.filename == '' + + +def test_load_config_python(): + config.load_config('reframe/core/settings.py') + + +def test_load_config_python_old_syntax(): + with pytest.raises(ReframeDeprecationWarning): + site_config = config.load_config( + 'unittests/resources/settings_old_syntax.py' + ) + + +def test_convert_old_config(): + converted = config.convert_old_config( + 'unittests/resources/settings_old_syntax.py' + ) + site_config = config.load_config(converted) + site_config.validate() + assert len(site_config.get('systems')) == 3 + + site_config.select_subconfig('testsys') + assert len(site_config.get('systems/0/partitions')) == 2 + assert len(site_config.get('modes')) == 1 + assert len(site_config['environments']) == 6 + + +def test_load_config_python_invalid(tmp_path): + pyfile = tmp_path / 'settings.py' + pyfile.write_text('x = 1\n') + with pytest.raises(ConfigError, + match=r'not a valid Python configuration file'): + config.load_config(pyfile) + + +def test_load_config_json(tmp_path): + import reframe.core.settings as settings + + json_file = tmp_path / 'settings.json' + json_file.write_text(json.dumps(settings.site_configuration, indent=4)) + site_config = config.load_config(json_file) + assert site_config.filename == json_file + + +def test_load_config_json_invalid_syntax(tmp_path): + json_file = tmp_path / 'settings.json' + json_file.write_text('foo') + with pytest.raises(ConfigError, match=r'invalid JSON syntax'): + config.load_config(json_file) + + +def test_load_config_unknown_file(tmp_path): + with pytest.raises(OSError): + config.load_config(tmp_path / 'foo.json') + + +def test_load_config_import_error(): + # If the configuration file is relative to ReFrame and ImportError is + # raised, which should be wrapped inside ConfigError + with pytest.raises(ConfigError, + match=r'could not load Python configuration file'): + config.load_config('reframe/core/foo.py') + + +def test_load_config_unknown_filetype(tmp_path): + import reframe.core.settings as settings + + json_file = tmp_path / 'foo' + json_file.write_text(json.dumps(settings.site_configuration, indent=4)) + with pytest.raises(ConfigError, match=r'unknown configuration file type'): + config.load_config(json_file) + + +def test_validate_fallback_config(): + site_config = config.load_config('reframe/core/settings.py') + site_config.validate() + + +def test_validate_unittest_config(): + site_config = config.load_config('unittests/resources/settings.py') + site_config.validate() + + +def test_validate_config_invalid_syntax(): + site_config = config.load_config('reframe/core/settings.py') + site_config['systems'][0]['name'] = 123 + with pytest.raises(ConfigError, + match=r'could not validate configuration file'): + site_config.validate() + + +def test_validate_config_duplicate_systems(): + site_config = config.load_config('reframe/core/settings.py') + site_config['systems'].append(site_config['systems'][0]) + with pytest.raises(ConfigError, + match=r"system 'generic' already defined"): + site_config.validate() + + +def test_validate_config_duplicate_partitions(): + site_config = config.load_config('reframe/core/settings.py') + site_config['systems'][0]['partitions'].append( + site_config['systems'][0]['partitions'][0] + ) + with pytest.raises(ConfigError, + match=r"partition 'default' already defined"): + site_config.validate() + + +def test_select_subconfig_autodetect(): + site_config = config.load_config('reframe/core/settings.py') + site_config.select_subconfig() + assert site_config['systems'][0]['name'] == 'generic' + + +def test_select_subconfig_autodetect_failure(): + site_config = config.load_config('reframe/core/settings.py') + site_config['systems'][0]['hostnames'] = ['$^'] + with pytest.raises( + ConfigError, + match=(r'could not find a configuration entry ' + 'for the current system') + ): + site_config.select_subconfig() + + +def test_select_subconfig_unknown_system(): + site_config = config.load_config('reframe/core/settings.py') + with pytest.raises( + ConfigError, + match=(r'could not find a configuration entry ' + 'for the requested system') + ): + site_config.select_subconfig('foo') + + +def test_select_subconfig_unknown_partition(): + site_config = config.load_config('reframe/core/settings.py') + with pytest.raises( + ConfigError, + match=(r'could not find a configuration entry ' + 'for the requested system/partition') + ): + site_config.select_subconfig('generic:foo') + + +def test_select_subconfig_no_logging(): + site_config = config.load_config('reframe/core/settings.py') + site_config['logging'][0]['target_systems'] = ['foo'] + with pytest.raises(ConfigError, match=r"section 'logging' not defined"): + site_config.select_subconfig() + + +def test_select_subconfig_no_environments(): + site_config = config.load_config('reframe/core/settings.py') + site_config['environments'][0]['target_systems'] = ['foo'] + with pytest.raises(ConfigError, + match=r"section 'environments' not defined"): + site_config.select_subconfig() + + +def test_select_subconfig_undefined_environment(): + site_config = config.load_config('reframe/core/settings.py') + site_config['systems'][0]['partitions'][0]['environs'] += ['foo', 'bar'] + with pytest.raises( + ConfigError, + match=r"environments ('foo', 'bar')|('bar', 'foo') are not defined" + ): + site_config.select_subconfig() + + +def test_select_subconfig(): + site_config = config.load_config('unittests/resources/settings.py') + site_config.select_subconfig('testsys') + assert len(site_config['systems']) == 1 + assert len(site_config['systems'][0]['partitions']) == 2 + assert len(site_config['modes']) == 1 + assert site_config.get('systems/0/name') == 'testsys' + assert site_config.get('systems/0/descr') == 'Fake system for unit tests' + assert site_config.get('systems/0/hostnames') == ['testsys'] + assert site_config.get('systems/0/prefix') == '.rfm_testing' + assert (site_config.get('systems/0/resourcesdir') == + '.rfm_testing/resources') + assert site_config.get('systems/0/modules') == ['foo/1.0'] + assert site_config.get('systems/0/variables') == [['FOO_CMD', 'foobar']] + assert site_config.get('systems/0/modules_system') == 'nomod' + assert site_config.get('systems/0/outputdir') == '' + assert site_config.get('systems/0/stagedir') == '' + assert len(site_config.get('systems/0/partitions')) == 2 + assert site_config.get('systems/0/partitions/@gpu/max_jobs') == 10 + assert site_config.get('modes/0/name') == 'unittest' + assert site_config.get('modes/@unittest/name') == 'unittest' + assert len(site_config.get('logging/0/handlers')) == 2 + assert len(site_config.get('logging/0/handlers_perflog')) == 1 + assert site_config.get('logging/0/handlers/0/timestamp') is False + assert site_config.get('logging/0/handlers/0/level') == 'debug' + assert site_config.get('logging/0/handlers/1/level') == 'info' + assert site_config.get('logging/0/handlers/2/level') is None + + site_config.select_subconfig('testsys:login') + assert len(site_config.get('systems/0/partitions')) == 1 + assert site_config.get('systems/0/partitions/0/scheduler') == 'local' + assert site_config.get('systems/0/partitions/0/launcher') == 'local' + assert (site_config.get('systems/0/partitions/0/environs') == + ['PrgEnv-cray', 'PrgEnv-gnu', 'builtin-gcc']) + assert site_config.get('systems/0/partitions/0/descr') == 'Login nodes' + assert site_config.get('systems/0/partitions/0/resources') == [] + assert site_config.get('systems/0/partitions/0/access') == [] + assert site_config.get('systems/0/partitions/0/container_platforms') == [] + assert site_config.get('systems/0/partitions/0/modules') == [] + assert site_config.get('systems/0/partitions/0/variables') == [] + assert site_config.get('systems/0/partitions/0/max_jobs') == 1 + assert len(site_config['environments']) == 6 + assert site_config.get('environments/@PrgEnv-gnu/cc') == 'gcc' + assert site_config.get('environments/0/cxx') == 'g++' + assert site_config.get('environments/@PrgEnv-cray/cc') == 'cc' + assert site_config.get('environments/1/cxx') == 'CC' + assert (site_config.get('environments/@PrgEnv-cray/modules') == + ['PrgEnv-cray']) + assert len(site_config.get('general')) == 1 + assert site_config.get('general/0/check_search_path') == ['a:b'] + + site_config.select_subconfig('testsys:gpu') + assert site_config.get('systems/0/partitions/@gpu/scheduler') == 'slurm' + assert site_config.get('systems/0/partitions/0/launcher') == 'srun' + assert (site_config.get('systems/0/partitions/0/environs') == + ['PrgEnv-gnu', 'builtin-gcc']) + assert site_config.get('systems/0/partitions/0/descr') == 'GPU partition' + assert len(site_config.get('systems/0/partitions/0/resources')) == 2 + assert (site_config.get('systems/0/partitions/0/resources/@gpu/name') == + 'gpu') + assert site_config.get('systems/0/partitions/0/modules') == ['foogpu'] + assert (site_config.get('systems/0/partitions/0/variables') == + [['FOO_GPU', 'yes']]) + assert site_config.get('systems/0/partitions/0/max_jobs') == 10 + assert len(site_config['environments']) == 6 + assert site_config.get('environments/@PrgEnv-gnu/cc') == 'cc' + assert site_config.get('environments/0/cxx') == 'CC' + assert site_config.get('general/0/check_search_path') == ['c:d'] + + # Test inexistent options + site_config.select_subconfig('testsys') + assert site_config.get('systems/1/name') is None + assert site_config.get('systems/0/partitions/gpu/name') is None + assert site_config.get('environments/0/foo') is None + + # Test misplaced slashes or empty option + assert site_config.get('systems/0/partitions/@gpu/launcher/') == 'srun' + assert site_config.get('/systems/0/partitions') is None + assert site_config.get('', 'foo') == 'foo' + assert site_config.get(None, 'foo') == 'foo' + + +def test_select_subconfig_optional_section_absent(): + site_config = config.load_config('reframe/core/settings.py') + site_config.select_subconfig() + assert site_config.get('general/0/colorize') is True + assert site_config.get('general/verbose') == 0 + + +def test_sticky_options(): + site_config = config.load_config('unittests/resources/settings.py') + site_config.select_subconfig('testsys:login') + site_config.add_sticky_option('environments/cc', 'clang') + site_config.add_sticky_option('modes/options', ['foo']) + assert site_config.get('environments/@PrgEnv-gnu/cc') == 'clang' + assert site_config.get('environments/@PrgEnv-cray/cc') == 'clang' + assert site_config.get('environments/@PrgEnv-cray/cxx') == 'CC' + assert site_config.get('modes/0/options') == ['foo'] + + # Remove the sticky options + site_config.remove_sticky_option('environments/cc') + site_config.remove_sticky_option('modes/options') + assert site_config.get('environments/@PrgEnv-gnu/cc') == 'gcc' + assert site_config.get('environments/@PrgEnv-cray/cc') == 'cc' + + +def test_system_create(): + site_config = config.load_config('unittests/resources/settings.py') + site_config.select_subconfig('testsys:gpu') + system = System.create(site_config) + assert system.name == 'testsys' + assert system.descr == 'Fake system for unit tests' + assert system.hostnames == ['testsys'] + assert system.modules_system.name == 'nomod' + assert system.preload_environ.modules == ['foo/1.0'] + assert system.preload_environ.variables == {'FOO_CMD': 'foobar'} + assert system.prefix == '.rfm_testing' + assert system.stagedir == '' + assert system.outputdir == '' + assert system.resourcesdir == '.rfm_testing/resources' + assert len(system.partitions) == 1 + + partition = system.partitions[0] + assert partition.name == 'gpu' + assert partition.fullname == 'testsys:gpu' + assert partition.descr == 'GPU partition' + assert partition.scheduler.registered_name == 'slurm' + assert partition.launcher.registered_name == 'srun' + assert partition.access == [] + assert partition.container_environs == {} + assert partition.local_env.modules == ['foogpu'] + assert partition.local_env.variables == {'FOO_GPU': 'yes'} + assert partition.max_jobs == 10 + assert len(partition.environs) == 2 + assert partition.environment('PrgEnv-gnu').cc == 'cc' + assert partition.environment('PrgEnv-gnu').cflags == [] + + # Check resource instantiation + resource_spec = partition.get_resource('gpu', num_gpus_per_node=16) + assert resource_spec == ['--gres=gpu:16'] + + resources_spec = partition.get_resource( + 'datawarp', capacity='100GB', stagein_src='/foo' + ) + assert resources_spec == ['#DW jobdw capacity=100GB', + '#DW stage_in source=/foo'] diff --git a/unittests/test_containers.py b/unittests/test_containers.py index 7356031c0f..3c162be403 100644 --- a/unittests/test_containers.py +++ b/unittests/test_containers.py @@ -87,9 +87,9 @@ def expected_cmd_with_run_opts(self): "image:tag bash -c 'cd /stagedir; cmd'") -class TestShifterNG(_ContainerPlatformTest, unittest.TestCase): +class TestShifter(_ContainerPlatformTest, unittest.TestCase): def create_container_platform(self): - return containers.ShifterNG() + return containers.Shifter() @property def expected_cmd_mount_points(self): @@ -109,7 +109,7 @@ def expected_cmd_with_run_opts(self): "--foo --bar image:tag bash -c 'cd /stagedir; cmd'") -class TestShifterNGLocalImage(TestShifterNG): +class TestShifterLocalImage(TestShifter): @property def expected_cmd_prepare(self): return [] @@ -120,9 +120,9 @@ def test_prepare_command(self): self.container_platform.emit_prepare_commands()) -class TestShifterNGWithMPI(TestShifterNG): +class TestShifterWithMPI(TestShifter): def create_container_platform(self): - ret = containers.ShifterNG() + ret = containers.Shifter() ret.with_mpi = True return ret diff --git a/unittests/test_dependencies.py b/unittests/test_dependencies.py new file mode 100644 index 0000000000..2d0ffacb6e --- /dev/null +++ b/unittests/test_dependencies.py @@ -0,0 +1,472 @@ +# Copyright 2016-2020 Swiss National Supercomputing Centre (CSCS/ETH Zurich) +# ReFrame Project Developers. See the top-level LICENSE file for details. +# +# SPDX-License-Identifier: BSD-3-Clause + +import itertools +import pytest + +import reframe as rfm +import reframe.core.runtime as rt +import reframe.frontend.dependency as dependency +import reframe.frontend.executors as executors +import reframe.utility as util +from reframe.core.environments import Environment +from reframe.core.exceptions import DependencyError +from reframe.frontend.loader import RegressionCheckLoader + +import unittests.fixtures as fixtures + + +class Node: + '''A node in the test case graph. + + It's simply a wrapper to a (test_name, partition, environment) tuple + that can interact seemlessly with a real test case. + It's meant for convenience in unit testing. + ''' + + def __init__(self, cname, pname, ename): + self.cname, self.pname, self.ename = cname, pname, ename + + def __eq__(self, other): + if isinstance(other, type(self)): + return (self.cname == other.cname and + self.pname == other.pname and + self.ename == other.ename) + + if isinstance(other, executors.TestCase): + return (self.cname == other.check.name and + self.pname == other.partition.fullname and + self.ename == other.environ.name) + + return NotImplemented + + def __hash__(self): + return hash(self.cname) ^ hash(self.pname) ^ hash(self.ename) + + def __repr__(self): + return 'Node(%r, %r, %r)' % (self.cname, self.pname, self.ename) + + +def has_edge(graph, src, dst): + return dst in graph[src] + + +def num_deps(graph, cname): + return sum(len(deps) for c, deps in graph.items() + if c.check.name == cname) + + +def in_degree(graph, node): + for v in graph.keys(): + if v == node: + return v.num_dependents + + +def find_check(name, checks): + for c in checks: + if c.name == name: + return c + + return None + + +def find_case(cname, ename, cases): + for c in cases: + if c.check.name == cname and c.environ.name == ename: + return c + + +@pytest.fixture +def temp_runtime(tmp_path): + def _temp_runtime(site_config, system=None, options={}): + options.update({'systems/prefix': tmp_path}) + with rt.temp_runtime(site_config, system, options): + yield rt.runtime + + yield _temp_runtime + + +@pytest.fixture +def exec_ctx(temp_runtime): + yield from temp_runtime(fixtures.TEST_CONFIG_FILE, 'sys0') + + +@pytest.fixture +def loader(): + return RegressionCheckLoader([ + 'unittests/resources/checks_unlisted/deps_simple.py' + ]) + + +def test_eq_hash(loader, exec_ctx): + cases = executors.generate_testcases(loader.load_all()) + case0 = find_case('Test0', 'e0', cases) + case1 = find_case('Test0', 'e1', cases) + case0_copy = case0.clone() + + assert case0 == case0_copy + assert hash(case0) == hash(case0_copy) + assert case1 != case0 + assert hash(case1) != hash(case0) + + +def test_build_deps(loader, exec_ctx): + checks = loader.load_all() + cases = executors.generate_testcases(checks) + + # Test calling getdep() before having built the graph + t = find_check('Test1_exact', checks) + with pytest.raises(DependencyError): + t.getdep('Test0', 'e0') + + # Build dependencies and continue testing + deps = dependency.build_deps(cases) + dependency.validate_deps(deps) + + # Check DEPEND_FULLY dependencies + assert num_deps(deps, 'Test1_fully') == 8 + for p in ['sys0:p0', 'sys0:p1']: + for e0 in ['e0', 'e1']: + for e1 in ['e0', 'e1']: + assert has_edge(deps, + Node('Test1_fully', p, e0), + Node('Test0', p, e1)) + + # Check DEPEND_BY_ENV + assert num_deps(deps, 'Test1_by_env') == 4 + assert num_deps(deps, 'Test1_default') == 4 + for p in ['sys0:p0', 'sys0:p1']: + for e in ['e0', 'e1']: + assert has_edge(deps, + Node('Test1_by_env', p, e), + Node('Test0', p, e)) + assert has_edge(deps, + Node('Test1_default', p, e), + Node('Test0', p, e)) + + # Check DEPEND_EXACT + assert num_deps(deps, 'Test1_exact') == 6 + for p in ['sys0:p0', 'sys0:p1']: + assert has_edge(deps, + Node('Test1_exact', p, 'e0'), + Node('Test0', p, 'e0')) + assert has_edge(deps, + Node('Test1_exact', p, 'e0'), + Node('Test0', p, 'e1')) + assert has_edge(deps, + Node('Test1_exact', p, 'e1'), + Node('Test0', p, 'e1')) + + # Check in-degree of Test0 + + # 2 from Test1_fully, + # 1 from Test1_by_env, + # 1 from Test1_exact, + # 1 from Test1_default + assert in_degree(deps, Node('Test0', 'sys0:p0', 'e0')) == 5 + assert in_degree(deps, Node('Test0', 'sys0:p1', 'e0')) == 5 + + # 2 from Test1_fully, + # 1 from Test1_by_env, + # 2 from Test1_exact, + # 1 from Test1_default + assert in_degree(deps, Node('Test0', 'sys0:p0', 'e1')) == 6 + assert in_degree(deps, Node('Test0', 'sys0:p1', 'e1')) == 6 + + # Pick a check to test getdep() + check_e0 = find_case('Test1_exact', 'e0', cases).check + check_e1 = find_case('Test1_exact', 'e1', cases).check + + with pytest.raises(DependencyError): + check_e0.getdep('Test0') + + # Set the current environment + check_e0._current_environ = Environment('e0') + check_e1._current_environ = Environment('e1') + + assert check_e0.getdep('Test0', 'e0').name == 'Test0' + assert check_e0.getdep('Test0', 'e1').name == 'Test0' + assert check_e1.getdep('Test0', 'e1').name == 'Test0' + with pytest.raises(DependencyError): + check_e0.getdep('TestX', 'e0') + + with pytest.raises(DependencyError): + check_e0.getdep('Test0', 'eX') + + with pytest.raises(DependencyError): + check_e1.getdep('Test0', 'e0') + + +def test_build_deps_unknown_test(loader, exec_ctx): + checks = loader.load_all() + + # Add some inexistent dependencies + test0 = find_check('Test0', checks) + for depkind in ('default', 'fully', 'by_env', 'exact'): + test1 = find_check('Test1_' + depkind, checks) + if depkind == 'default': + test1.depends_on('TestX') + elif depkind == 'exact': + test1.depends_on('TestX', rfm.DEPEND_EXACT, {'e0': ['e0']}) + elif depkind == 'fully': + test1.depends_on('TestX', rfm.DEPEND_FULLY) + elif depkind == 'by_env': + test1.depends_on('TestX', rfm.DEPEND_BY_ENV) + + with pytest.raises(DependencyError): + dependency.build_deps(executors.generate_testcases(checks)) + + +def test_build_deps_unknown_target_env(loader, exec_ctx): + checks = loader.load_all() + + # Add some inexistent dependencies + test0 = find_check('Test0', checks) + test1 = find_check('Test1_default', checks) + test1.depends_on('Test0', rfm.DEPEND_EXACT, {'e0': ['eX']}) + with pytest.raises(DependencyError): + dependency.build_deps(executors.generate_testcases(checks)) + + +def test_build_deps_unknown_source_env(loader, exec_ctx): + checks = loader.load_all() + + # Add some inexistent dependencies + test0 = find_check('Test0', checks) + test1 = find_check('Test1_default', checks) + test1.depends_on('Test0', rfm.DEPEND_EXACT, {'eX': ['e0']}) + + # Unknown source is ignored, because it might simply be that the test + # is not executed for eX + deps = dependency.build_deps(executors.generate_testcases(checks)) + assert num_deps(deps, 'Test1_default') == 4 + + +def test_build_deps_empty(exec_ctx): + assert {} == dependency.build_deps([]) + + +@pytest.fixture +def make_test(): + class MyTest(rfm.RegressionTest): + def __init__(self, name): + self.name = name + self.valid_systems = ['*'] + self.valid_prog_environs = ['*'] + self.executable = 'echo' + self.executable_opts = [name] + + def _make_test(name): + return MyTest(name) + + return _make_test + + +def test_valid_deps(make_test, exec_ctx): + # + # t0 +-->t5<--+ + # ^ | | + # | | | + # +-->t1<--+ t6 t7 + # | | ^ + # t2<------t3 | + # ^ ^ | + # | | t8 + # +---t4---+ + # + t0 = make_test('t0') + t1 = make_test('t1') + t2 = make_test('t2') + t3 = make_test('t3') + t4 = make_test('t4') + t5 = make_test('t5') + t6 = make_test('t6') + t7 = make_test('t7') + t8 = make_test('t8') + t1.depends_on('t0') + t2.depends_on('t1') + t3.depends_on('t1') + t3.depends_on('t2') + t4.depends_on('t2') + t4.depends_on('t3') + t6.depends_on('t5') + t7.depends_on('t5') + t8.depends_on('t7') + dependency.validate_deps( + dependency.build_deps( + executors.generate_testcases([t0, t1, t2, t3, t4, + t5, t6, t7, t8]) + ) + ) + + +def test_cyclic_deps(make_test, exec_ctx): + # + # t0 +-->t5<--+ + # ^ | | + # | | | + # +-->t1<--+ t6 t7 + # | | | ^ + # t2 | t3 | + # ^ | ^ | + # | v | t8 + # +---t4---+ + # + t0 = make_test('t0') + t1 = make_test('t1') + t2 = make_test('t2') + t3 = make_test('t3') + t4 = make_test('t4') + t5 = make_test('t5') + t6 = make_test('t6') + t7 = make_test('t7') + t8 = make_test('t8') + t1.depends_on('t0') + t1.depends_on('t4') + t2.depends_on('t1') + t3.depends_on('t1') + t4.depends_on('t2') + t4.depends_on('t3') + t6.depends_on('t5') + t7.depends_on('t5') + t8.depends_on('t7') + deps = dependency.build_deps( + executors.generate_testcases([t0, t1, t2, t3, t4, + t5, t6, t7, t8]) + ) + + with pytest.raises(DependencyError) as exc_info: + dependency.validate_deps(deps) + + assert ('t4->t2->t1->t4' in str(exc_info.value) or + 't2->t1->t4->t2' in str(exc_info.value) or + 't1->t4->t2->t1' in str(exc_info.value) or + 't1->t4->t3->t1' in str(exc_info.value) or + 't4->t3->t1->t4' in str(exc_info.value) or + 't3->t1->t4->t3' in str(exc_info.value)) + + +def test_cyclic_deps_by_env(make_test, exec_ctx): + t0 = make_test('t0') + t1 = make_test('t1') + t1.depends_on('t0', rfm.DEPEND_EXACT, {'e0': ['e0']}) + t0.depends_on('t1', rfm.DEPEND_EXACT, {'e1': ['e1']}) + deps = dependency.build_deps( + executors.generate_testcases([t0, t1]) + ) + with pytest.raises(DependencyError) as exc_info: + dependency.validate_deps(deps) + + assert ('t1->t0->t1' in str(exc_info.value) or + 't0->t1->t0' in str(exc_info.value)) + + +def test_validate_deps_empty(exec_ctx): + dependency.validate_deps({}) + + +def assert_topological_order(cases, graph): + cases_order = [] + visited_tests = set() + tests = util.OrderedSet() + for c in cases: + check, part, env = c + cases_order.append((check.name, part.fullname, env.name)) + tests.add(check.name) + visited_tests.add(check.name) + + # Assert that all dependencies of c have been visited before + for d in graph[c]: + if d not in cases: + # dependency points outside the subgraph + continue + + assert d.check.name in visited_tests + + # Check the order of systems and prog. environments + # We are checking against all possible orderings + valid_orderings = [] + for partitions in itertools.permutations(['sys0:p0', 'sys0:p1']): + for environs in itertools.permutations(['e0', 'e1']): + ordering = [] + for t in tests: + for p in partitions: + for e in environs: + ordering.append((t, p, e)) + + valid_orderings.append(ordering) + + assert cases_order in valid_orderings + + +def test_toposort(make_test, exec_ctx): + # + # t0 +-->t5<--+ + # ^ | | + # | | | + # +-->t1<--+ t6 t7 + # | | ^ + # t2<------t3 | + # ^ ^ | + # | | t8 + # +---t4---+ + # + t0 = make_test('t0') + t1 = make_test('t1') + t2 = make_test('t2') + t3 = make_test('t3') + t4 = make_test('t4') + t5 = make_test('t5') + t6 = make_test('t6') + t7 = make_test('t7') + t8 = make_test('t8') + t1.depends_on('t0') + t2.depends_on('t1') + t3.depends_on('t1') + t3.depends_on('t2') + t4.depends_on('t2') + t4.depends_on('t3') + t6.depends_on('t5') + t7.depends_on('t5') + t8.depends_on('t7') + deps = dependency.build_deps( + executors.generate_testcases([t0, t1, t2, t3, t4, + t5, t6, t7, t8]) + ) + cases = dependency.toposort(deps) + assert_topological_order(cases, deps) + + +def test_toposort_subgraph(make_test, exec_ctx): + # + # t0 + # ^ + # | + # +-->t1<--+ + # | | + # t2<------t3 + # ^ ^ + # | | + # +---t4---+ + # + t0 = make_test('t0') + t1 = make_test('t1') + t2 = make_test('t2') + t3 = make_test('t3') + t4 = make_test('t4') + t1.depends_on('t0') + t2.depends_on('t1') + t3.depends_on('t1') + t3.depends_on('t2') + t4.depends_on('t2') + t4.depends_on('t3') + full_deps = dependency.build_deps( + executors.generate_testcases([t0, t1, t2, t3, t4]) + ) + partial_deps = dependency.build_deps( + executors.generate_testcases([t3, t4]), full_deps + ) + cases = dependency.toposort(partial_deps, is_subgraph=True) + assert_topological_order(cases, partial_deps) diff --git a/unittests/test_environments.py b/unittests/test_environments.py index bbfdb58963..cccb988e2d 100644 --- a/unittests/test_environments.py +++ b/unittests/test_environments.py @@ -8,9 +8,9 @@ import unittest import reframe.core.environments as env +import reframe.core.runtime as rt import reframe.utility.os_ext as os_ext import unittests.fixtures as fixtures -from reframe.core.runtime import runtime from reframe.core.exceptions import EnvironError @@ -27,7 +27,7 @@ def setup_modules_system(self): if not fixtures.has_sane_modules_system(): pytest.skip('no modules system configured') - self.modules_system = runtime().modules_system + self.modules_system = rt.runtime().modules_system self.modules_system.searchpath_add(fixtures.TEST_MODULES) # Always add a base module; this is a workaround for the modules @@ -69,14 +69,14 @@ def test_setup(self): self.environ.variables['_var3'] == '${_var1}' def test_environ_snapshot(self): - env.load(self.environ, self.environ_other) + rt.loadenv(self.environ, self.environ_other) self.environ_save.restore() assert self.environ_save == env.snapshot() - assert not self.environ.is_loaded - assert not self.environ_other.is_loaded + assert not rt.is_env_loaded(self.environ) + assert not rt.is_env_loaded(self.environ_other) def test_load_restore(self): - snapshot, _ = env.load(self.environ) + snapshot, _ = rt.loadenv(self.environ) os.environ['_var0'] == 'val1' os.environ['_var1'] == 'val1' os.environ['_var2'] == 'val1' @@ -84,46 +84,46 @@ def test_load_restore(self): if fixtures.has_sane_modules_system(): self.assertModulesLoaded(self.environ.modules) - assert self.environ.is_loaded + assert rt.is_env_loaded(self.environ) snapshot.restore() self.environ_save == env.snapshot() os.environ['_var0'], 'val0' if fixtures.has_sane_modules_system(): assert not self.modules_system.is_module_loaded('testmod_foo') - assert not self.environ.is_loaded + assert not rt.is_env_loaded(self.environ) @fixtures.switch_to_user_runtime def test_temp_environment(self): self.setup_modules_system() - with env.temp_environment( + with rt.temp_environment( ['testmod_foo'], {'_var0': 'val2', '_var3': 'val3'} ) as environ: - assert environ.is_loaded + assert rt.is_env_loaded(environ) - assert not environ.is_loaded + assert not rt.is_env_loaded(environ) @fixtures.switch_to_user_runtime def test_load_already_present(self): self.setup_modules_system() self.modules_system.load_module('testmod_boo') - snapshot, _ = env.load(self.environ) + snapshot, _ = rt.loadenv(self.environ) snapshot.restore() assert self.modules_system.is_module_loaded('testmod_boo') def test_load_non_overlapping(self): e0 = env.Environment(name='e0', variables=[('a', '1'), ('b', '2')]) e1 = env.Environment(name='e1', variables=[('c', '3'), ('d', '4')]) - env.load(e0, e1) - assert e0.is_loaded - assert e1.is_loaded + rt.loadenv(e0, e1) + assert rt.is_env_loaded(e0) + assert rt.is_env_loaded(e1) def test_load_overlapping(self): e0 = env.Environment(name='e0', variables=[('a', '1'), ('b', '2')]) e1 = env.Environment(name='e1', variables=[('b', '3'), ('c', '4')]) - env.load(e0, e1) - assert not e0.is_loaded - assert e1.is_loaded + rt.loadenv(e0, e1) + assert not rt.is_env_loaded(e0) + assert rt.is_env_loaded(e1) def test_equal(self): env1 = env.Environment('env1', modules=['foo', 'bar']) @@ -147,7 +147,7 @@ def test_conflicting_environments(self): envfoo = env.Environment(name='envfoo', modules=['testmod_foo', 'testmod_boo']) envbar = env.Environment(name='envbar', modules=['testmod_bar']) - env.load(envfoo, envbar) + rt.loadenv(envfoo, envbar) for m in envbar.modules: assert self.modules_system.is_module_loaded(m) @@ -159,7 +159,7 @@ def test_conflict_environ_after_module_load(self): self.setup_modules_system() self.modules_system.load_module('testmod_foo') envfoo = env.Environment(name='envfoo', modules=['testmod_foo']) - snapshot, _ = env.load(envfoo) + snapshot, _ = rt.loadenv(envfoo) snapshot.restore() assert self.modules_system.is_module_loaded('testmod_foo') @@ -168,17 +168,17 @@ def test_conflict_environ_after_module_force_load(self): self.setup_modules_system() self.modules_system.load_module('testmod_foo') envbar = env.Environment(name='envbar', modules=['testmod_bar']) - snapshot, _ = env.load(envbar) + snapshot, _ = rt.loadenv(envbar) snapshot.restore() assert self.modules_system.is_module_loaded('testmod_foo') def test_immutability(self): # Check emit_load_commands() - _, commands = env.load(self.environ) + _, commands = rt.loadenv(self.environ) # Try to modify the returned list of commands commands.append('foo') - assert 'foo' not in env.load(self.environ)[1] + assert 'foo' not in rt.loadenv(self.environ)[1] # Test ProgEnvironment prgenv = env.ProgEnvironment('foo_prgenv') @@ -213,14 +213,14 @@ def test_immutability(self): @fixtures.switch_to_user_runtime def test_emit_load_commands(self): self.setup_modules_system() - rt = runtime() + ms = rt.runtime().modules_system expected_commands = [ - rt.modules_system.emit_load_commands('testmod_foo')[0], + ms.emit_load_commands('testmod_foo')[0], 'export _var0=val1', 'export _var2=$_var0', 'export _var3=${_var1}', ] - assert expected_commands == env.emit_load_commands(self.environ) + assert expected_commands == rt.emit_loadenv_commands(self.environ) @fixtures.switch_to_user_runtime def test_emit_load_commands_with_confict(self): @@ -228,12 +228,12 @@ def test_emit_load_commands_with_confict(self): # Load a conflicting module self.modules_system.load_module('testmod_bar') - rt = runtime() + ms = rt.runtime().modules_system expected_commands = [ - rt.modules_system.emit_unload_commands('testmod_bar')[0], - rt.modules_system.emit_load_commands('testmod_foo')[0], + ms.emit_unload_commands('testmod_bar')[0], + ms.emit_load_commands('testmod_foo')[0], 'export _var0=val1', 'export _var2=$_var0', 'export _var3=${_var1}', ] - assert expected_commands == env.emit_load_commands(self.environ) + assert expected_commands == rt.emit_loadenv_commands(self.environ) diff --git a/unittests/test_launchers.py b/unittests/test_launchers.py index c809ff6768..0988ddf2d4 100644 --- a/unittests/test_launchers.py +++ b/unittests/test_launchers.py @@ -7,7 +7,7 @@ import unittest import reframe.core.launchers as launchers -from reframe.core.launchers.registry import getlauncher +from reframe.core.backends import getlauncher from reframe.core.schedulers import Job, JobScheduler diff --git a/unittests/test_loader.py b/unittests/test_loader.py index b8e63fea44..5368d2e6d4 100644 --- a/unittests/test_loader.py +++ b/unittests/test_loader.py @@ -21,9 +21,6 @@ def setUp(self): self.loader_with_path = RegressionCheckLoader( ['unittests/resources/checks', 'unittests/foobar'], ignore_conflicts=True) - self.loader_with_prefix = RegressionCheckLoader( - load_path=['bad'], - prefix=os.path.abspath('unittests/resources/checks')) def test_load_file_relative(self): checks = self.loader.load_from_file( @@ -46,10 +43,6 @@ def test_load_all(self): checks = self.loader_with_path.load_all() assert 11 == len(checks) - def test_load_all_with_prefix(self): - checks = self.loader_with_prefix.load_all() - assert 1 == len(checks) - def test_load_new_syntax(self): checks = self.loader.load_from_file( 'unittests/resources/checks_unlisted/good.py') diff --git a/unittests/test_logging.py b/unittests/test_logging.py index 6ea020d358..042f0cda4f 100644 --- a/unittests/test_logging.py +++ b/unittests/test_logging.py @@ -3,12 +3,13 @@ # # SPDX-License-Identifier: BSD-3-Clause +import copy import logging import logging.handlers import os import pytest -import sys import re +import sys import tempfile import time import unittest @@ -16,10 +17,12 @@ import reframe as rfm import reframe.core.logging as rlog +import reframe.core.runtime as rt +import reframe.core.settings as settings +import reframe.utility as util from reframe.core.exceptions import ConfigError, ReframeError -from reframe.core.launchers.registry import getlauncher +from reframe.core.backends import (getlauncher, getscheduler) from reframe.core.schedulers import Job -from reframe.core.schedulers.registry import getscheduler class _FakeCheck(rfm.RegressionTest): @@ -143,324 +146,257 @@ def test_rfc3339_timezone_wrong_directive(self): assert self.found_in_logfile(':z') -class TestLoggingConfiguration(unittest.TestCase): - def setUp(self): - tmpfd, self.logfile = tempfile.mkstemp(dir='.') - os.close(tmpfd) - self.logging_config = { - 'level': 'INFO', - 'handlers': [ - { - 'type': 'file', - 'name': self.logfile, - 'level': 'WARNING', - 'format': '[%(asctime)s] %(levelname)s: ' - '%(check_name)s: %(message)s', - 'datefmt': '%F', - 'append': True, - } - ] - } - self.check = _FakeCheck() +@pytest.fixture +def temp_runtime(tmp_path): + def _temp_runtime(logging_config): + site_config = copy.deepcopy(settings.site_configuration) + site_config['logging'] = [logging_config] + with tempfile.NamedTemporaryFile(mode='w+t', dir=str(tmp_path), + suffix='.py', delete=False) as fp: + fp.write(f'site_configuration = {util.ppretty(site_config)}') - def tearDown(self): - if os.path.exists(self.logfile): - os.remove(self.logfile) + with rt.temp_runtime(fp.name): + yield rt.runtime() - def found_in_logfile(self, string): - for handler in rlog.getlogger().logger.handlers: - handler.flush() - handler.close() + return _temp_runtime - found = False - with open(self.logfile, 'rt') as f: - found = string in f.read() - return found +@pytest.fixture +def logfile(tmp_path): + return str(tmp_path / 'test.log') - def close_handlers(self): - for h in rlog.getlogger().logger.handlers: - h.close() - def flush_handlers(self): - for h in rlog.getlogger().logger.handlers: - h.flush() +@pytest.fixture +def basic_config(temp_runtime, logfile): + yield from temp_runtime({ + 'level': 'info', + 'handlers': [ + { + 'type': 'file', + 'name': logfile, + 'level': 'warning', + 'format': '[%(asctime)s] %(levelname)s: ' + '%(check_name)s: %(message)s', + 'datefmt': '%F', + 'append': True, + }, + ], + 'handlers_perflog': [] + }) - def test_valid_level(self): - rlog.configure_logging(self.logging_config) - assert rlog.INFO == rlog.getlogger().getEffectiveLevel() - def test_no_handlers(self): - del self.logging_config['handlers'] - with pytest.raises(ValueError): - rlog.configure_logging(self.logging_config) +def _flush_handlers(): + for h in rlog.getlogger().logger.handlers: + h.flush() - def test_empty_handlers(self): - self.logging_config['handlers'] = [] - with pytest.raises(ValueError): - rlog.configure_logging(self.logging_config) - def test_handler_level(self): - rlog.configure_logging(self.logging_config) - rlog.getlogger().info('foo') - rlog.getlogger().warning('bar') +def _close_handlers(): + for h in rlog.getlogger().logger.handlers: + h.close() - assert not self.found_in_logfile('foo') - assert self.found_in_logfile('bar') - def test_handler_append(self): - rlog.configure_logging(self.logging_config) - rlog.getlogger().warning('foo') - self.close_handlers() +def _found_in_logfile(string, filename): + _flush_handlers() + _close_handlers() + found = False + with open(filename, 'rt') as fp: + found = string in fp.read() - # Reload logger - rlog.configure_logging(self.logging_config) - rlog.getlogger().warning('bar') + return found - assert self.found_in_logfile('foo') - assert self.found_in_logfile('bar') - def test_handler_noappend(self): - self.logging_config = { - 'level': 'INFO', - 'handlers': [ - { - 'type': 'file', - 'name': self.logfile, - 'level': 'WARNING', - 'format': '[%(asctime)s] %(levelname)s: %(message)s', - 'datefmt': '%F', - 'append': False, - } - ] - } +def test_valid_level(basic_config): + rlog.configure_logging(rt.runtime().site_config) + assert rlog.INFO == rlog.getlogger().getEffectiveLevel() - rlog.configure_logging(self.logging_config) - rlog.getlogger().warning('foo') - self.close_handlers() - # Reload logger - rlog.configure_logging(self.logging_config) - rlog.getlogger().warning('bar') +def test_handler_level(basic_config, logfile): + rlog.configure_logging(rt.runtime().site_config) + rlog.getlogger().info('foo') + rlog.getlogger().warning('bar') + assert not _found_in_logfile('foo', logfile) + assert _found_in_logfile('bar', logfile) - assert not self.found_in_logfile('foo') - assert self.found_in_logfile('bar') - def test_date_format(self): - rlog.configure_logging(self.logging_config) - rlog.getlogger().warning('foo') - assert self.found_in_logfile(datetime.now().strftime('%F')) +def test_handler_append(basic_config, logfile): + rlog.configure_logging(rt.runtime().site_config) + rlog.getlogger().warning('foo') + _close_handlers() - def test_unknown_handler(self): - self.logging_config = { - 'level': 'INFO', - 'handlers': [ - {'type': 'stream', 'name': 'stderr'}, - {'type': 'foo'} - ], - } - with pytest.raises(ConfigError): - rlog.configure_logging(self.logging_config) + # Reload logger + rlog.configure_logging(rt.runtime().site_config) + rlog.getlogger().warning('bar') - def test_handler_syntax_no_type(self): - self.logging_config = { - 'level': 'INFO', - 'handlers': [{'name': 'stderr'}] - } - with pytest.raises(ConfigError): - rlog.configure_logging(self.logging_config) - - def test_handler_convert_syntax(self): - old_syntax = { - self.logfile: { - 'level': 'INFO', - 'format': '%(message)s', - 'append': False, - }, - '&1': { - 'level': 'INFO', - 'format': '%(message)s' - }, - '&2': { - 'level': 'ERROR', - 'format': '%(message)s' - } - } - - new_syntax = [ - { - 'type': 'file', - 'name': self.logfile, - 'level': 'INFO', - 'format': '%(message)s', - 'append': False - }, - { - 'type': 'stream', - 'name': 'stdout', - 'level': 'INFO', - 'format': '%(message)s' - }, - { - 'type': 'stream', - 'name': 'stderr', - 'level': 'ERROR', - 'format': '%(message)s' - } - ] - - self.assertCountEqual(new_syntax, - rlog._convert_handler_syntax(old_syntax)) - - def test_stream_handler_stdout(self): - self.logging_config = { - 'level': 'INFO', - 'handlers': [{'type': 'stream', 'name': 'stdout'}], - } - rlog.configure_logging(self.logging_config) - raw_logger = rlog.getlogger().logger - assert len(raw_logger.handlers) == 1 - handler = raw_logger.handlers[0] - - assert isinstance(handler, logging.StreamHandler) - assert handler.stream == sys.stdout - - def test_stream_handler_stderr(self): - self.logging_config = { - 'level': 'INFO', - 'handlers': [{'type': 'stream', 'name': 'stderr'}], - } + assert _found_in_logfile('foo', logfile) + assert _found_in_logfile('bar', logfile) - rlog.configure_logging(self.logging_config) - raw_logger = rlog.getlogger().logger - assert len(raw_logger.handlers) == 1 - handler = raw_logger.handlers[0] - assert isinstance(handler, logging.StreamHandler) - assert handler.stream == sys.stderr - - def test_multiple_handlers(self): - self.logging_config = { - 'level': 'INFO', +def test_handler_noappend(temp_runtime, logfile): + runtime = temp_runtime( + { + 'level': 'info', 'handlers': [ - {'type': 'stream', 'name': 'stderr'}, - {'type': 'file', 'name': self.logfile}, - {'type': 'syslog', 'address': '/dev/log'} - ], - } - rlog.configure_logging(self.logging_config) - assert len(rlog.getlogger().logger.handlers) == 3 - - def test_file_handler_timestamp(self): - self.logging_config['handlers'][0]['timestamp'] = '%F' - rlog.configure_logging(self.logging_config) - rlog.getlogger().warning('foo') - logfile = '%s_%s' % (self.logfile, datetime.now().strftime('%F')) - assert os.path.exists(logfile) - os.remove(logfile) - - def test_file_handler_syntax_no_name(self): - self.logging_config = { - 'level': 'INFO', - 'handlers': [ - {'type': 'file'} + { + 'type': 'file', + 'name': logfile, + 'level': 'warning', + 'format': '[%(asctime)s] %(levelname)s: %(message)s', + 'datefmt': '%F', + 'append': False, + } ], + 'handlers_perflog': [] } - with pytest.raises(ConfigError): - rlog.configure_logging(self.logging_config) - def test_stream_handler_unknown_stream(self): - self.logging_config = { - 'level': 'INFO', - 'handlers': [ - {'type': 'stream', 'name': 'foo'}, - ], - } - with pytest.raises(ConfigError): - rlog.configure_logging(self.logging_config) - - def test_syslog_handler(self): - import platform - - if platform.system() == 'Linux': - addr = '/dev/log' - elif platform.system() == 'Darwin': - addr = '/dev/run/syslog' - else: - pytest.skip() - - self.logging_config = { - 'level': 'INFO', - 'handlers': [{'type': 'syslog', 'address': addr}] - } - rlog.getlogger().info('foo') + ) + next(runtime) - def test_syslog_handler_no_address(self): - self.logging_config = { - 'level': 'INFO', - 'handlers': [{'type': 'syslog'}] - } - with pytest.raises(ConfigError): - rlog.configure_logging(self.logging_config) + rlog.configure_logging(rt.runtime().site_config) + rlog.getlogger().warning('foo') + _close_handlers() - def test_syslog_handler_unknown_facility(self): - self.logging_config = { - 'level': 'INFO', - 'handlers': [{'type': 'syslog', 'facility': 'foo'}] - } - with pytest.raises(ConfigError): - rlog.configure_logging(self.logging_config) + # Reload logger + rlog.configure_logging(rt.runtime().site_config) + rlog.getlogger().warning('bar') - def test_syslog_handler_unknown_socktype(self): - self.logging_config = { - 'level': 'INFO', - 'handlers': [{'type': 'syslog', 'socktype': 'foo'}] - } - with pytest.raises(ConfigError): - rlog.configure_logging(self.logging_config) + assert not _found_in_logfile('foo', logfile) + assert _found_in_logfile('bar', logfile) - def test_global_noconfig(self): - # This is to test the case when no configuration is set, but since the - # order the unit tests are invoked is arbitrary, we emulate the - # 'no-config' state by passing `None` to `configure_logging()` - rlog.configure_logging(None) - assert rlog.getlogger() is rlog.null_logger +def test_date_format(basic_config, logfile): + rlog.configure_logging(rt.runtime().site_config) + rlog.getlogger().warning('foo') + assert _found_in_logfile(datetime.now().strftime('%F'), logfile) - def test_global_config(self): - rlog.configure_logging(self.logging_config) - assert rlog.getlogger() is not rlog.null_logger - def test_logging_context(self): - rlog.configure_logging(self.logging_config) - with rlog.logging_context() as logger: - assert logger is rlog.getlogger() - assert logger is not rlog.null_logger - rlog.getlogger().error('error from context') +@pytest.fixture(params=['stdout', 'stderr']) +def stream(request): + return request.param - assert self.found_in_logfile('reframe') - assert self.found_in_logfile('error from context') - def test_logging_context_check(self): - rlog.configure_logging(self.logging_config) - with rlog.logging_context(check=self.check): - rlog.getlogger().error('error from context') +def test_stream_handler(temp_runtime, logfile, stream): + runtime = temp_runtime({ + 'level': 'info', + 'handlers': [{'type': 'stream', 'name': stream}], + 'handlers_perflog': [] + }) + next(runtime) + rlog.configure_logging(rt.runtime().site_config) + raw_logger = rlog.getlogger().logger + assert len(raw_logger.handlers) == 1 + handler = raw_logger.handlers[0] - rlog.getlogger().error('error outside context') - assert self.found_in_logfile( - '_FakeCheck: %s: error from context' % sys.argv[0]) - assert self.found_in_logfile( - 'reframe: %s: error outside context' % sys.argv[0]) + assert isinstance(handler, logging.StreamHandler) + stream = sys.stdout if stream == 'stdout' else sys.stderr + assert handler.stream == stream - def test_logging_context_error(self): - rlog.configure_logging(self.logging_config) - try: - with rlog.logging_context(level=rlog.ERROR): - raise ReframeError('error from context') - pytest.fail('logging_context did not propagate the exception') - except ReframeError: - pass +def test_multiple_handlers(temp_runtime, logfile): + runtime = temp_runtime({ + 'level': 'info', + 'handlers': [ + {'type': 'stream', 'name': 'stderr'}, + {'type': 'file', 'name': logfile}, + {'type': 'syslog', 'address': '/dev/log'} + ], + 'handlers_perflog': [] + }) + next(runtime) + rlog.configure_logging(rt.runtime().site_config) + assert len(rlog.getlogger().logger.handlers) == 3 - assert self.found_in_logfile('reframe') - assert self.found_in_logfile('error from context') + +def test_file_handler_timestamp(temp_runtime, logfile): + runtime = temp_runtime({ + 'level': 'info', + 'handlers': [ + { + 'type': 'file', + 'name': logfile, + 'level': 'warning', + 'format': '[%(asctime)s] %(levelname)s: ' + '%(check_name)s: %(message)s', + 'datefmt': '%F', + 'timestamp': '%F', + 'append': True, + }, + ], + 'handlers_perflog': [] + }) + next(runtime) + rlog.configure_logging(rt.runtime().site_config) + rlog.getlogger().warning('foo') + base, ext = os.path.splitext(logfile) + filename = f"{base}_{datetime.now().strftime('%F')}.log" + assert os.path.exists(filename) + + +def test_syslog_handler(temp_runtime): + import platform + + if platform.system() == 'Linux': + addr = '/dev/log' + elif platform.system() == 'Darwin': + addr = '/dev/run/syslog' + else: + pytest.skip('unknown system platform') + + runtime = temp_runtime({ + 'level': 'info', + 'handlers': [{'type': 'syslog', 'address': addr}], + 'handlers_perflog': [] + }) + next(runtime) + rlog.configure_logging(rt.runtime().site_config) + rlog.getlogger().info('foo') + + +def test_global_noconfig(): + # This is to test the case when no configuration is set, but since the + # order the unit tests are invoked is arbitrary, we emulate the + # 'no-config' state by passing `None` to `configure_logging()` + + rlog.configure_logging(None) + assert rlog.getlogger() is rlog.null_logger + + +def test_global_config(basic_config): + rlog.configure_logging(rt.runtime().site_config) + assert rlog.getlogger() is not rlog.null_logger + + +def test_logging_context(basic_config, logfile): + rlog.configure_logging(rt.runtime().site_config) + with rlog.logging_context() as logger: + assert logger is rlog.getlogger() + assert logger is not rlog.null_logger + rlog.getlogger().error('error from context') + + assert _found_in_logfile('reframe', logfile) + assert _found_in_logfile('error from context', logfile) + + +def test_logging_context_check(basic_config, logfile): + rlog.configure_logging(rt.runtime().site_config) + with rlog.logging_context(check=_FakeCheck()): + rlog.getlogger().error('error from context') + + rlog.getlogger().error('error outside context') + assert _found_in_logfile(f'_FakeCheck: {sys.argv[0]}: error from context', + logfile) + assert _found_in_logfile(f'reframe: {sys.argv[0]}: error outside context', + logfile) + + +def test_logging_context_error(basic_config, logfile): + rlog.configure_logging(rt.runtime().site_config) + try: + with rlog.logging_context(level=rlog.ERROR): + raise ReframeError('error from context') + + pytest.fail('logging_context did not propagate the exception') + except ReframeError: + pass + + assert _found_in_logfile('reframe', logfile) + assert _found_in_logfile('error from context', logfile) diff --git a/unittests/test_pipeline.py b/unittests/test_pipeline.py index e0c010921a..1c2ff1812d 100644 --- a/unittests/test_pipeline.py +++ b/unittests/test_pipeline.py @@ -4,10 +4,9 @@ # SPDX-License-Identifier: BSD-3-Clause import os +import pathlib import pytest import re -import tempfile -import unittest import reframe as rfm import reframe.core.runtime as rt @@ -21,26 +20,6 @@ from unittests.resources.checks.hellocheck import HelloTest -def _setup_local_execution(): - partition = rt.runtime().system.partition('login') - environ = partition.environment('builtin-gcc') - return partition, environ - - -def _setup_remote_execution(scheduler=None): - partition = fixtures.partition_with_scheduler(scheduler) - if partition is None: - pytest.skip('job submission not supported') - - try: - environ = partition.environs[0] - except IndexError: - pytest.skip('no environments configured for partition: %s' % - partition.fullname) - - return partition, environ - - def _run(test, partition, prgenv): test.setup(partition, prgenv) test.compile() @@ -52,1008 +31,993 @@ def _run(test, partition, prgenv): test.cleanup(remove_files=True) -def _cray_cle_version(): - completed = os_ext.run_command('cat /etc/opt/cray/release/cle-release') - matched = re.match(r'^RELEASE=(\S+)', completed.stdout) - if matched is None: - return None +def load_test(testfile): + loader = RegressionCheckLoader(['unittests/resources/checks']) + return loader.load_from_file(testfile) - return matched.group(1) +@pytest.fixture +def temp_runtime(tmp_path): + def _temp_runtime(config_file, system=None, options={}): + options.update({'systems/prefix': str(tmp_path)}) + with rt.temp_runtime(config_file, system, options): + yield rt.runtime() -class TestRegressionTest(unittest.TestCase): - def setUp(self): - self.partition, self.prgenv = _setup_local_execution() - self.loader = RegressionCheckLoader(['unittests/resources/checks']) - - # Set runtime prefix - rt.runtime().resources.prefix = tempfile.mkdtemp(dir='unittests') - - def tearDown(self): - os_ext.rmtree(rt.runtime().resources.prefix) - os_ext.rmtree('.rfm_testing', ignore_errors=True) - - def replace_prefix(self, filename, new_prefix): - basename = os.path.basename(filename) - return os.path.join(new_prefix, basename) - - def keep_files_list(self, test, compile_only=False): - ret = [self.replace_prefix(sn.evaluate(test.stdout), test.outputdir), - self.replace_prefix(sn.evaluate(test.stderr), test.outputdir)] - - if not compile_only: - ret.append(self.replace_prefix(test.job.script_filename, - test.outputdir)) - - ret.extend([self.replace_prefix(f, test.outputdir) - for f in test.keep_files]) - return ret - - def test_environ_setup(self): - test = self.loader.load_from_file( - 'unittests/resources/checks/hellocheck.py')[0] - - # Use test environment for the regression check - test.valid_prog_environs = [self.prgenv.name] - test.modules = ['testmod_foo'] - test.variables = {'_FOO_': '1', '_BAR_': '2'} - test.local = True - - test.setup(self.partition, self.prgenv) - - for k in test.variables.keys(): - assert k not in os.environ - - def _run_test(self, test, compile_only=False): - _run(test, self.partition, self.prgenv) - assert not os.path.exists(test.stagedir) - for f in self.keep_files_list(test, compile_only): - assert os.path.exists(f) - - @fixtures.switch_to_user_runtime - def test_hellocheck(self): - self.partition, self.prgenv = _setup_remote_execution() - test = self.loader.load_from_file( - 'unittests/resources/checks/hellocheck.py')[0] - - # Use test environment for the regression check - test.valid_prog_environs = [self.prgenv.name] - self._run_test(test) - - @fixtures.switch_to_user_runtime - def test_hellocheck_make(self): - self.partition, self.prgenv = _setup_remote_execution() - test = self.loader.load_from_file( - 'unittests/resources/checks/hellocheck_make.py')[0] - - # Use test environment for the regression check - test.valid_prog_environs = [self.prgenv.name] - self._run_test(test) - - def test_hellocheck_local(self): - test = self.loader.load_from_file( - 'unittests/resources/checks/hellocheck.py')[0] - - # Use test environment for the regression check - test.valid_prog_environs = [self.prgenv.name] - - # Test also the prebuild/postbuild functionality - test.prebuild_cmd = ['touch prebuild', 'mkdir prebuild_dir'] - test.postbuild_cmd = ['touch postbuild', 'mkdir postbuild_dir'] - test.keep_files = ['prebuild', 'postbuild', - 'prebuild_dir', 'postbuild_dir'] - - # Force local execution of the test - test.local = True - self._run_test(test) - - def test_hellocheck_local_prepost_run(self): - @sn.sanity_function - def stagedir(test): - return test.stagedir - - test = self.loader.load_from_file( - 'unittests/resources/checks/hellocheck.py')[0] - - # Use test environment for the regression check - test.valid_prog_environs = [self.prgenv.name] - - # Test also the prebuild/postbuild functionality - test.pre_run = ['echo prerun: `pwd`'] - test.post_run = ['echo postrun: `pwd`'] - pre_run_path = sn.extractsingle(r'^prerun: (\S+)', test.stdout, 1) - post_run_path = sn.extractsingle(r'^postrun: (\S+)', test.stdout, 1) - test.sanity_patterns = sn.all([ - sn.assert_eq(stagedir(test), pre_run_path), - sn.assert_eq(stagedir(test), post_run_path), - ]) - - # Force local execution of the test - test.local = True - self._run_test(test) - - def test_hellocheck_local_prepost_run_in_setup(self): - def custom_setup(obj, partition, environ, **job_opts): - super(obj.__class__, obj).setup(partition, environ, **job_opts) - obj.pre_run = ['echo Prerunning cmd from setup phase'] - obj.post_run = ['echo Postruning cmd from setup phase'] - - test = self.loader.load_from_file( - 'unittests/resources/checks/hellocheck.py')[0] - - # Monkey patch the setup method of the test - test.setup = custom_setup.__get__(test) - - # Use test environment for the regression check - test.valid_prog_environs = ['*'] - - test.sanity_patterns = sn.all([ - sn.assert_found(r'^Prerunning cmd from setup phase', test.stdout), - sn.assert_found(r'Hello, World\!', test.stdout), - sn.assert_found(r'^Postruning cmd from setup phase', test.stdout) - ]) - - # Force local execution of the test - test.local = True - self._run_test(test) - - def test_run_only_sanity(self): - @fixtures.custom_prefix('unittests/resources/checks') - class MyTest(rfm.RunOnlyRegressionTest): - def __init__(self): - self.executable = './hello.sh' - self.executable_opts = ['Hello, World!'] - self.local = True - self.valid_prog_environs = ['*'] - self.valid_systems = ['*'] - self.sanity_patterns = sn.assert_found( - r'Hello, World\!', self.stdout) + yield _temp_runtime - self._run_test(MyTest()) - def test_run_only_no_srcdir(self): - @fixtures.custom_prefix('foo/bar/') - class MyTest(rfm.RunOnlyRegressionTest): - def __init__(self): - self.executable = 'echo' - self.executable_opts = ['hello'] - self.valid_prog_environs = ['*'] - self.valid_systems = ['*'] - self.sanity_patterns = sn.assert_found(r'hello', self.stdout) +@pytest.fixture +def generic_system(temp_runtime): + yield from temp_runtime(fixtures.TEST_CONFIG_FILE, 'generic') - test = MyTest() - assert test.sourcesdir is None - self._run_test(MyTest()) - def test_compile_only_failure(self): - @fixtures.custom_prefix('unittests/resources/checks') - class MyTest(rfm.CompileOnlyRegressionTest): - def __init__(self): - self.sourcepath = 'compiler_failure.c' - self.valid_prog_environs = ['*'] - self.valid_systems = ['*'] +@pytest.fixture +def testsys_system(temp_runtime): + yield from temp_runtime(fixtures.TEST_CONFIG_FILE, 'testsys') - test = MyTest() - test.setup(self.partition, self.prgenv) - test.compile() - with pytest.raises(BuildError): - test.compile_wait() - def test_compile_only_warning(self): - @fixtures.custom_prefix('unittests/resources/checks') - class MyTest(rfm.RunOnlyRegressionTest): - def __init__(self): - self.build_system = 'SingleSource' - self.build_system.srcfile = 'compiler_warning.c' - self.build_system.cflags = ['-Wall'] - self.valid_prog_environs = ['*'] - self.valid_systems = ['*'] - self.sanity_patterns = sn.assert_found(r'warning', self.stderr) - - self._run_test(MyTest(), compile_only=True) - - @rt.switch_runtime(fixtures.TEST_SITE_CONFIG, 'testsys') - def test_supports_system(self): - test = self.loader.load_from_file( - 'unittests/resources/checks/hellocheck.py')[0] - - test.valid_systems = ['*'] - assert test.supports_system('gpu') - assert test.supports_system('login') - assert test.supports_system('testsys:gpu') - assert test.supports_system('testsys:login') - - test.valid_systems = ['*:*'] - assert test.supports_system('gpu') - assert test.supports_system('login') - assert test.supports_system('testsys:gpu') - assert test.supports_system('testsys:login') - - test.valid_systems = ['testsys'] - assert test.supports_system('gpu') - assert test.supports_system('login') - assert test.supports_system('testsys:gpu') - assert test.supports_system('testsys:login') - - test.valid_systems = ['testsys:gpu'] - assert test.supports_system('gpu') - assert not test.supports_system('login') - assert test.supports_system('testsys:gpu') - assert not test.supports_system('testsys:login') - - test.valid_systems = ['testsys:login'] - assert not test.supports_system('gpu') - assert test.supports_system('login') - assert not test.supports_system('testsys:gpu') - assert test.supports_system('testsys:login') - - test.valid_systems = ['foo'] - assert not test.supports_system('gpu') - assert not test.supports_system('login') - assert not test.supports_system('testsys:gpu') - assert not test.supports_system('testsys:login') - - test.valid_systems = ['*:gpu'] - assert test.supports_system('testsys:gpu') - assert test.supports_system('foo:gpu') - assert not test.supports_system('testsys:cpu') - assert not test.supports_system('testsys:login') - - test.valid_systems = ['testsys:*'] - assert test.supports_system('testsys:login') - assert test.supports_system('gpu') - assert not test.supports_system('foo:gpu') - - def test_supports_environ(self): - test = self.loader.load_from_file( - 'unittests/resources/checks/hellocheck.py')[0] - - test.valid_prog_environs = ['*'] - assert test.supports_environ('foo1') - assert test.supports_environ('foo-env') - assert test.supports_environ('*') - - def test_sourcesdir_none(self): - @fixtures.custom_prefix('unittests/resources/checks') - class MyTest(rfm.RegressionTest): - def __init__(self): - self.sourcesdir = None - self.valid_prog_environs = ['*'] - self.valid_systems = ['*'] +@pytest.fixture +def user_system(temp_runtime): + if fixtures.USER_CONFIG_FILE: + yield from temp_runtime(fixtures.USER_CONFIG_FILE, + fixtures.USER_SYSTEM) + else: + yield generic_system - with pytest.raises(ReframeError): - self._run_test(MyTest()) - def test_sourcesdir_build_system(self): - @fixtures.custom_prefix('unittests/resources/checks') - class MyTest(rfm.RegressionTest): - def __init__(self): - self.build_system = 'Make' - self.sourcepath = 'code' - self.executable = './code/hello' - self.local = True - self.valid_systems = ['*'] - self.valid_prog_environs = ['*'] - self.sanity_patterns = sn.assert_found(r'Hello, World\!', - self.stdout) +@pytest.fixture +def hellotest(): + yield load_test('unittests/resources/checks/hellocheck.py')[0] - self._run_test(MyTest()) - def test_sourcesdir_none_generated_sources(self): - @fixtures.custom_prefix('unittests/resources/checks') - class MyTest(rfm.RegressionTest): - def __init__(self): - self.sourcesdir = None - self.prebuild_cmd = [ - "printf '#include \\n int main(){ " - "printf(\"Hello, World!\\\\n\"); return 0; }' > hello.c" - ] - self.executable = './hello' - self.sourcepath = 'hello.c' - self.local = True - self.valid_systems = ['*'] - self.valid_prog_environs = ['*'] - self.sanity_patterns = sn.assert_found(r'Hello, World\!', - self.stdout) +@pytest.fixture +def local_exec_ctx(generic_system): + partition = fixtures.partition_by_name('default') + environ = fixtures.environment_by_name('builtin-gcc', partition) + yield partition, environ - self._run_test(MyTest()) - def test_sourcesdir_none_compile_only(self): - @fixtures.custom_prefix('unittests/resources/checks') - class MyTest(rfm.CompileOnlyRegressionTest): - def __init__(self): - self.sourcesdir = None - self.valid_prog_environs = ['*'] - self.valid_systems = ['*'] +@pytest.fixture +def local_user_exec_ctx(user_system): + partition = fixtures.partition_by_scheduler('local') + if partition is None: + pytest.skip('no local jobs are supported') - with pytest.raises(BuildError): - self._run_test(MyTest()) + try: + environ = partition.environs[0] + except IndexError: + pytest.skip('no environments configured for partition: %s' % + partition.fullname) - def test_sourcesdir_none_run_only(self): - @fixtures.custom_prefix('unittests/resources/checks') - class MyTest(rfm.RunOnlyRegressionTest): - def __init__(self): - self.sourcesdir = None - self.executable = 'echo' - self.executable_opts = ["Hello, World!"] - self.local = True - self.valid_prog_environs = ['*'] - self.valid_systems = ['*'] - self.sanity_patterns = sn.assert_found(r'Hello, World\!', - self.stdout) + yield partition, environ - self._run_test(MyTest()) - def test_sourcepath_abs(self): - @fixtures.custom_prefix('unittests/resources/checks') - class MyTest(rfm.CompileOnlyRegressionTest): - def __init__(self): - self.valid_prog_environs = ['*'] - self.valid_systems = ['*'] +@pytest.fixture +def remote_exec_ctx(user_system): + partition = fixtures.partition_by_scheduler() + if partition is None: + pytest.skip('job submission not supported') - test = MyTest() - test.setup(self.partition, self.prgenv) - test.sourcepath = '/usr/src' - with pytest.raises(PipelineError): - test.compile() + try: + environ = partition.environs[0] + except IndexError: + pytest.skip('no environments configured for partition: %s' % + partition.fullname) - def test_sourcepath_upref(self): - @fixtures.custom_prefix('unittests/resources/checks') - class MyTest(rfm.CompileOnlyRegressionTest): - def __init__(self): - self.valid_prog_environs = ['*'] - self.valid_systems = ['*'] + yield partition, environ - test = MyTest() - test.setup(self.partition, self.prgenv) - test.sourcepath = '../hellosrc' - with pytest.raises(PipelineError): - test.compile() - @rt.switch_runtime(fixtures.TEST_SITE_CONFIG, 'testsys') - def test_extra_resources(self): - @fixtures.custom_prefix('unittests/resources/checks') - class MyTest(HelloTest): - def __init__(self): - super().__init__() - self.name = type(self).__name__ - self.executable = os.path.join('.', self.name) - self.local = True - - @rfm.run_after('setup') - def set_resources(self): - test.extra_resources = { - 'gpu': {'num_gpus_per_node': 2}, - 'datawarp': {'capacity': '100GB', - 'stagein_src': test.stagedir} - } - test.job.options += ['--foo'] +@pytest.fixture +def remote_exec_ctx(user_system): + partition = fixtures.partition_by_scheduler() + if partition is None: + pytest.skip('job submission not supported') - test = MyTest() - partition = rt.runtime().system.partition('gpu') - environ = partition.environment('builtin-gcc') - _run(test, partition, environ) - expected_job_options = ['--gres=gpu:2', - '#DW jobdw capacity=100GB', - '#DW stage_in source=%s' % test.stagedir, - '--foo'] - self.assertCountEqual(expected_job_options, test.job.options) + try: + environ = partition.environs[0] + except IndexError: + pytest.skip('no environments configured for partition: %s' % + partition.fullname) + yield partition, environ -class TestHooks(unittest.TestCase): - def setUp(self): - self.partition = rt.runtime().system.partition('login') - self.prgenv = self.partition.environment('builtin-gcc') - # Set runtime prefix - rt.runtime().resources.prefix = tempfile.mkdtemp(dir='unittests') +@pytest.fixture +def container_remote_exec_ctx(remote_exec_ctx): + def _container_exec_ctx(platform): + partition = remote_exec_ctx[0] + if platform not in partition.container_environs.keys(): + pytest.skip('%s is not configured on the system' % platform) - def tearDown(self): - os_ext.rmtree(rt.runtime().resources.prefix) + yield from remote_exec_ctx - def test_setup_hooks(self): - @fixtures.custom_prefix('unittests/resources/checks') - class MyTest(HelloTest): - def __init__(self): - super().__init__() - self.name = type(self).__name__ - self.executable = os.path.join('.', self.name) - - @rfm.run_before('setup') - def prefoo(self): - assert self.current_environ is None - os.environ['_RFM_PRE_SETUP'] = 'foo' - - @rfm.run_after('setup') - def postfoo(self): - assert self.current_environ is not None - os.environ['_RFM_POST_SETUP'] = 'foo' - - test = MyTest() - _run(test, self.partition, self.prgenv) - assert '_RFM_PRE_SETUP' in os.environ - assert '_RFM_POST_SETUP' in os.environ - - def test_setup_hooks_in_compile_only_test(self): - @fixtures.custom_prefix('unittests/resources/checks') - class MyTest(rfm.CompileOnlyRegressionTest): - def __init__(self): - self.name = 'hellocheck_compile' - self.valid_systems = ['*'] - self.valid_prog_environs = ['*'] - self.sourcepath = 'hello.c' - self.executable = os.path.join('.', self.name) - self.sanity_patterns = sn.assert_found('.*', self.stdout) - self.count = 0 - - @rfm.run_before('setup') - def presetup(self): - self.count += 1 - - @rfm.run_after('setup') - def postsetup(self): - self.count += 1 - - test = MyTest() - _run(test, self.partition, self.prgenv) - assert test.count == 2 - - def test_compile_hooks(self): - @fixtures.custom_prefix('unittests/resources/checks') - class MyTest(HelloTest): - def __init__(self): - super().__init__() - self.name = type(self).__name__ - self.executable = os.path.join('.', self.name) + return _container_exec_ctx - @rfm.run_before('compile') - def setflags(self): - os.environ['_RFM_PRE_COMPILE'] = 'FOO' - @rfm.run_after('compile') - def check_executable(self): - exec_file = os.path.join(self.stagedir, self.executable) +@pytest.fixture +def container_local_exec_ctx(local_user_exec_ctx): + def _container_exec_ctx(platform): + partition = local_user_exec_ctx[0] + if platform not in partition.container_environs.keys(): + pytest.skip('%s is not configured on the system' % platform) - # Make sure that this hook is executed after compile_wait() - assert os.path.exists(exec_file) + yield from local_user_exec_ctx + + return _container_exec_ctx + + +def test_environ_setup(hellotest, local_exec_ctx): + # Use test environment for the regression check + hellotest.variables = {'_FOO_': '1', '_BAR_': '2'} + hellotest.setup(*local_exec_ctx) + for k in hellotest.variables.keys(): + assert k not in os.environ + + +def test_hellocheck(hellotest, remote_exec_ctx): + _run(hellotest, *remote_exec_ctx) + + +def test_hellocheck_make(remote_exec_ctx): + test = load_test('unittests/resources/checks/hellocheck_make.py')[0] + _run(test, *remote_exec_ctx) + + +def test_hellocheck_local(hellotest, local_exec_ctx): + # Test also the prebuild/postbuild functionality + hellotest.prebuild_cmd = ['touch prebuild', 'mkdir prebuild_dir'] + hellotest.postbuild_cmd = ['touch postbuild', 'mkdir postbuild_dir'] + hellotest.keep_files = ['prebuild', 'postbuild', + 'prebuild_dir', 'postbuild_dir'] + + # Force local execution of the test; just for testing .local + hellotest.local = True + _run(hellotest, *local_exec_ctx) + must_keep = [ + hellotest.stdout.evaluate(), + hellotest.stderr.evaluate(), + hellotest.build_stdout.evaluate(), + hellotest.build_stderr.evaluate(), + hellotest.job.script_filename, + *hellotest.keep_files + ] + for f in must_keep: + assert os.path.exists(os.path.join(hellotest.outputdir, f)) + + +def test_hellocheck_local_prepost_run(hellotest, local_exec_ctx): + @sn.sanity_function + def stagedir(test): + return test.stagedir + + # Test also the prebuild/postbuild functionality + hellotest.pre_run = ['echo prerun: `pwd`'] + hellotest.post_run = ['echo postrun: `pwd`'] + pre_run_path = sn.extractsingle(r'^prerun: (\S+)', hellotest.stdout, 1) + post_run_path = sn.extractsingle(r'^postrun: (\S+)', hellotest.stdout, 1) + hellotest.sanity_patterns = sn.all([ + sn.assert_eq(stagedir(hellotest), pre_run_path), + sn.assert_eq(stagedir(hellotest), post_run_path), + ]) + _run(hellotest, *local_exec_ctx) + + +def test_run_only_sanity(local_exec_ctx): + @fixtures.custom_prefix('unittests/resources/checks') + class MyTest(rfm.RunOnlyRegressionTest): + def __init__(self): + self.executable = './hello.sh' + self.executable_opts = ['Hello, World!'] + self.local = True + self.valid_prog_environs = ['*'] + self.valid_systems = ['*'] + self.sanity_patterns = sn.assert_found( + r'Hello, World\!', self.stdout) + + _run(MyTest(), *local_exec_ctx) + + +def test_run_only_no_srcdir(local_exec_ctx): + @fixtures.custom_prefix('foo/bar/') + class MyTest(rfm.RunOnlyRegressionTest): + def __init__(self): + self.executable = 'echo' + self.executable_opts = ['hello'] + self.valid_prog_environs = ['*'] + self.valid_systems = ['*'] + self.sanity_patterns = sn.assert_found(r'hello', self.stdout) + + test = MyTest() + assert test.sourcesdir is None + _run(test, *local_exec_ctx) + + +def test_compile_only_failure(local_exec_ctx): + @fixtures.custom_prefix('unittests/resources/checks') + class MyTest(rfm.CompileOnlyRegressionTest): + def __init__(self): + self.sourcepath = 'compiler_failure.c' + self.valid_prog_environs = ['*'] + self.valid_systems = ['*'] + + test = MyTest() + test.setup(*local_exec_ctx) + test.compile() + with pytest.raises(BuildError): + test.compile_wait() + + +def test_compile_only_warning(local_exec_ctx): + @fixtures.custom_prefix('unittests/resources/checks') + class MyTest(rfm.RunOnlyRegressionTest): + def __init__(self): + self.build_system = 'SingleSource' + self.build_system.srcfile = 'compiler_warning.c' + self.build_system.cflags = ['-Wall'] + self.valid_prog_environs = ['*'] + self.valid_systems = ['*'] + self.sanity_patterns = sn.assert_found(r'warning', self.stderr) + + _run(MyTest(), *local_exec_ctx) + + +def test_supports_system(hellotest, testsys_system): + hellotest.valid_systems = ['*'] + assert hellotest.supports_system('gpu') + assert hellotest.supports_system('login') + assert hellotest.supports_system('testsys:gpu') + assert hellotest.supports_system('testsys:login') + + hellotest.valid_systems = ['*:*'] + assert hellotest.supports_system('gpu') + assert hellotest.supports_system('login') + assert hellotest.supports_system('testsys:gpu') + assert hellotest.supports_system('testsys:login') + + hellotest.valid_systems = ['testsys'] + assert hellotest.supports_system('gpu') + assert hellotest.supports_system('login') + assert hellotest.supports_system('testsys:gpu') + assert hellotest.supports_system('testsys:login') + + hellotest.valid_systems = ['testsys:gpu'] + assert hellotest.supports_system('gpu') + assert not hellotest.supports_system('login') + assert hellotest.supports_system('testsys:gpu') + assert not hellotest.supports_system('testsys:login') + + hellotest.valid_systems = ['testsys:login'] + assert not hellotest.supports_system('gpu') + assert hellotest.supports_system('login') + assert not hellotest.supports_system('testsys:gpu') + assert hellotest.supports_system('testsys:login') + + hellotest.valid_systems = ['foo'] + assert not hellotest.supports_system('gpu') + assert not hellotest.supports_system('login') + assert not hellotest.supports_system('testsys:gpu') + assert not hellotest.supports_system('testsys:login') + + hellotest.valid_systems = ['*:gpu'] + assert hellotest.supports_system('testsys:gpu') + assert hellotest.supports_system('foo:gpu') + assert not hellotest.supports_system('testsys:cpu') + assert not hellotest.supports_system('testsys:login') + + hellotest.valid_systems = ['testsys:*'] + assert hellotest.supports_system('testsys:login') + assert hellotest.supports_system('gpu') + assert not hellotest.supports_system('foo:gpu') + + +def test_supports_environ(hellotest, generic_system): + hellotest.valid_prog_environs = ['*'] + assert hellotest.supports_environ('foo1') + assert hellotest.supports_environ('foo-env') + assert hellotest.supports_environ('*') + + +def test_sourcesdir_none(local_exec_ctx): + @fixtures.custom_prefix('unittests/resources/checks') + class MyTest(rfm.RegressionTest): + def __init__(self): + self.sourcesdir = None + self.valid_prog_environs = ['*'] + self.valid_systems = ['*'] + + with pytest.raises(ReframeError): + _run(MyTest(), *local_exec_ctx) + + +def test_sourcesdir_build_system(local_exec_ctx): + @fixtures.custom_prefix('unittests/resources/checks') + class MyTest(rfm.RegressionTest): + def __init__(self): + self.build_system = 'Make' + self.sourcepath = 'code' + self.executable = './code/hello' + self.valid_systems = ['*'] + self.valid_prog_environs = ['*'] + self.sanity_patterns = sn.assert_found(r'Hello, World\!', + self.stdout) + + _run(MyTest(), *local_exec_ctx) + + +def test_sourcesdir_none_generated_sources(local_exec_ctx): + @fixtures.custom_prefix('unittests/resources/checks') + class MyTest(rfm.RegressionTest): + def __init__(self): + self.sourcesdir = None + self.prebuild_cmd = [ + "printf '#include \\n int main(){ " + "printf(\"Hello, World!\\\\n\"); return 0; }' > hello.c" + ] + self.executable = './hello' + self.sourcepath = 'hello.c' + self.valid_systems = ['*'] + self.valid_prog_environs = ['*'] + self.sanity_patterns = sn.assert_found(r'Hello, World\!', + self.stdout) + + _run(MyTest(), *local_exec_ctx) + + +def test_sourcesdir_none_compile_only(local_exec_ctx): + @fixtures.custom_prefix('unittests/resources/checks') + class MyTest(rfm.CompileOnlyRegressionTest): + def __init__(self): + self.sourcesdir = None + self.valid_prog_environs = ['*'] + self.valid_systems = ['*'] + + with pytest.raises(BuildError): + _run(MyTest(), *local_exec_ctx) + + +def test_sourcesdir_none_run_only(local_exec_ctx): + @fixtures.custom_prefix('unittests/resources/checks') + class MyTest(rfm.RunOnlyRegressionTest): + def __init__(self): + self.sourcesdir = None + self.executable = 'echo' + self.executable_opts = ["Hello, World!"] + self.valid_prog_environs = ['*'] + self.valid_systems = ['*'] + self.sanity_patterns = sn.assert_found(r'Hello, World\!', + self.stdout) + + _run(MyTest(), *local_exec_ctx) + + +def test_sourcepath_abs(local_exec_ctx): + @fixtures.custom_prefix('unittests/resources/checks') + class MyTest(rfm.CompileOnlyRegressionTest): + def __init__(self): + self.valid_prog_environs = ['*'] + self.valid_systems = ['*'] + + test = MyTest() + test.setup(*local_exec_ctx) + test.sourcepath = '/usr/src' + with pytest.raises(PipelineError): + test.compile() - def test_run_hooks(self): - @fixtures.custom_prefix('unittests/resources/checks') - class MyTest(HelloTest): - def __init__(self): - super().__init__() - self.name = type(self).__name__ - self.executable = os.path.join('.', self.name) - @rfm.run_before('run') - def setflags(self): - self.post_run = ['echo hello > greetings.txt'] +def test_sourcepath_upref(local_exec_ctx): + @fixtures.custom_prefix('unittests/resources/checks') + class MyTest(rfm.CompileOnlyRegressionTest): + def __init__(self): + self.valid_prog_environs = ['*'] + self.valid_systems = ['*'] - @rfm.run_after('run') - def check_executable(self): - outfile = os.path.join(self.stagedir, 'greetings.txt') + test = MyTest() + test.setup(*local_exec_ctx) + test.sourcepath = '../hellosrc' + with pytest.raises(PipelineError): + test.compile() - # Make sure that this hook is executed after wait() - assert os.path.exists(outfile) - test = MyTest() - _run(test, self.partition, self.prgenv) +def test_extra_resources(testsys_system): + @fixtures.custom_prefix('unittests/resources/checks') + class MyTest(HelloTest): + def __init__(self): + super().__init__() + self.name = type(self).__name__ + self.executable = os.path.join('.', self.name) + self.local = True + + @rfm.run_after('setup') + def set_resources(self): + test.extra_resources = { + 'gpu': {'num_gpus_per_node': 2}, + 'datawarp': {'capacity': '100GB', + 'stagein_src': test.stagedir} + } + test.job.options += ['--foo'] + + test = MyTest() + partition = fixtures.partition_by_name('gpu') + environ = partition.environment('builtin-gcc') + _run(test, partition, environ) + expected_job_options = {'--gres=gpu:2', + '#DW jobdw capacity=100GB', + '#DW stage_in source=%s' % test.stagedir, + '--foo'} + assert expected_job_options == set(test.job.options) + + +def test_setup_hooks(local_exec_ctx): + @fixtures.custom_prefix('unittests/resources/checks') + class MyTest(HelloTest): + def __init__(self): + super().__init__() + self.name = type(self).__name__ + self.executable = os.path.join('.', self.name) + self.count = 0 + + @rfm.run_before('setup') + def prefoo(self): + assert self.current_environ is None + self.count += 1 + + @rfm.run_after('setup') + def postfoo(self): + assert self.current_environ is not None + self.count += 1 + + test = MyTest() + _run(test, *local_exec_ctx) + assert test.count == 2 + + +def test_compile_hooks(local_exec_ctx): + @fixtures.custom_prefix('unittests/resources/checks') + class MyTest(HelloTest): + def __init__(self): + super().__init__() + self.name = type(self).__name__ + self.executable = os.path.join('.', self.name) + self.count = 0 + + @rfm.run_before('compile') + def setflags(self): + self.count += 1 + + @rfm.run_after('compile') + def check_executable(self): + exec_file = os.path.join(self.stagedir, self.executable) + + # Make sure that this hook is executed after compile_wait() + assert os.path.exists(exec_file) + + test = MyTest() + _run(test, *local_exec_ctx) + assert test.count == 1 + + +def test_run_hooks(local_exec_ctx): + @fixtures.custom_prefix('unittests/resources/checks') + class MyTest(HelloTest): + def __init__(self): + super().__init__() + self.name = type(self).__name__ + self.executable = os.path.join('.', self.name) + + @rfm.run_before('run') + def setflags(self): + self.post_run = ['echo hello > greetings.txt'] + + @rfm.run_after('run') + def check_executable(self): + outfile = os.path.join(self.stagedir, 'greetings.txt') + + # Make sure that this hook is executed after wait() + assert os.path.exists(outfile) + + _run(MyTest(), *local_exec_ctx) + + +def test_multiple_hooks(local_exec_ctx): + @fixtures.custom_prefix('unittests/resources/checks') + class MyTest(HelloTest): + def __init__(self): + super().__init__() + self.name = type(self).__name__ + self.executable = os.path.join('.', self.name) + self.var = 0 + + @rfm.run_after('setup') + def x(self): + self.var += 1 + + @rfm.run_after('setup') + def y(self): + self.var += 1 + + @rfm.run_after('setup') + def z(self): + self.var += 1 + + test = MyTest() + _run(test, *local_exec_ctx) + assert test.var == 3 + + +def test_stacked_hooks(local_exec_ctx): + @fixtures.custom_prefix('unittests/resources/checks') + class MyTest(HelloTest): + def __init__(self): + super().__init__() + self.name = type(self).__name__ + self.executable = os.path.join('.', self.name) + self.var = 0 + + @rfm.run_before('setup') + @rfm.run_after('setup') + @rfm.run_after('compile') + def x(self): + self.var += 1 + + test = MyTest() + _run(test, *local_exec_ctx) + assert test.var == 3 + + +def test_inherited_hooks(local_exec_ctx): + @fixtures.custom_prefix('unittests/resources/checks') + class BaseTest(HelloTest): + def __init__(self): + super().__init__() + self.name = type(self).__name__ + self.executable = os.path.join('.', self.name) + self.var = 0 + + @rfm.run_after('setup') + def x(self): + self.var += 1 + + class C(rfm.RegressionTest): + @rfm.run_before('run') + def y(self): + self.foo = 1 + + class DerivedTest(BaseTest, C): + @rfm.run_after('setup') + def z(self): + self.var += 1 + + class MyTest(DerivedTest): + pass + + test = MyTest() + _run(test, *local_exec_ctx) + assert test.var == 2 + assert test.foo == 1 + + +def test_overriden_hooks(local_exec_ctx): + @fixtures.custom_prefix('unittests/resources/checks') + class BaseTest(HelloTest): + def __init__(self): + super().__init__() + self.name = type(self).__name__ + self.executable = os.path.join('.', self.name) + self.var = 0 + self.foo = 0 + + @rfm.run_after('setup') + def x(self): + self.var += 1 + + @rfm.run_before('setup') + def y(self): + self.foo += 1 + + class DerivedTest(BaseTest): + @rfm.run_after('setup') + def x(self): + self.var += 5 + + class MyTest(DerivedTest): + @rfm.run_before('setup') + def y(self): + self.foo += 10 + + test = MyTest() + _run(test, *local_exec_ctx) + assert test.var == 5 + assert test.foo == 10 + + +def test_require_deps(local_exec_ctx): + import reframe.frontend.dependency as dependency + import reframe.frontend.executors as executors + + @fixtures.custom_prefix('unittests/resources/checks') + class T0(HelloTest): + def __init__(self): + super().__init__() + self.name = type(self).__name__ + self.executable = os.path.join('.', self.name) + self.x = 1 + + @fixtures.custom_prefix('unittests/resources/checks') + class T1(HelloTest): + def __init__(self): + super().__init__() + self.name = type(self).__name__ + self.executable = os.path.join('.', self.name) + self.depends_on('T0') + + @rfm.require_deps + def sety(self, T0): + self.y = T0().x + 1 + + @rfm.run_before('run') + @rfm.require_deps + def setz(self, T0): + self.z = T0().x + 2 + + cases = executors.generate_testcases([T0(), T1()]) + deps = dependency.build_deps(cases) + for c in dependency.toposort(deps): + _run(*c) + + for c in cases: + t = c.check + if t.name == 'T0': + assert t.x == 1 + elif t.name == 'T1': + assert t.y == 2 + assert t.z == 3 + + +def test_regression_test_name(): + class MyTest(rfm.RegressionTest): + def __init__(self, a, b): + self.a = a + self.b = b + + test = MyTest(1, 2) + assert os.path.abspath(os.path.dirname(__file__)) == test.prefix + assert 'test_regression_test_name..MyTest_1_2' == test.name + + +def test_strange_test_names(): + class C: + def __init__(self, a): + self.a = a + + def __repr__(self): + return 'C(%s)' % self.a + + class MyTest(rfm.RegressionTest): + def __init__(self, a, b): + self.a = a + self.b = b + + test = MyTest('(a*b+c)/12', C(33)) + assert ('test_strange_test_names..MyTest__a_b_c__12_C_33_' == + test.name) + + +def test_name_user_inheritance(): + class MyBaseTest(rfm.RegressionTest): + def __init__(self, a, b): + self.a = a + self.b = b + + class MyTest(MyBaseTest): + def __init__(self): + super().__init__(1, 2) + + test = MyTest() + assert 'test_name_user_inheritance..MyTest' == test.name + + +def test_name_runonly_test(): + class MyTest(rfm.RunOnlyRegressionTest): + def __init__(self, a, b): + self.a = a + self.b = b + + test = MyTest(1, 2) + assert os.path.abspath(os.path.dirname(__file__)) == test.prefix + assert 'test_name_runonly_test..MyTest_1_2' == test.name + + +def test_name_compileonly_test(): + class MyTest(rfm.CompileOnlyRegressionTest): + def __init__(self, a, b): + self.a = a + self.b = b + + test = MyTest(1, 2) + assert os.path.abspath(os.path.dirname(__file__)) == test.prefix + assert 'test_name_compileonly_test..MyTest_1_2' == test.name + + +def test_registration_of_tests(): + import sys + import unittests.resources.checks_unlisted.good as mod + + checks = mod._rfm_gettests() + assert 13 == len(checks) + assert [mod.MyBaseTest(0, 0), + mod.MyBaseTest(0, 1), + mod.MyBaseTest(1, 0), + mod.MyBaseTest(1, 1), + mod.MyBaseTest(2, 0), + mod.MyBaseTest(2, 1), + mod.AnotherBaseTest(0, 0), + mod.AnotherBaseTest(0, 1), + mod.AnotherBaseTest(1, 0), + mod.AnotherBaseTest(1, 1), + mod.AnotherBaseTest(2, 0), + mod.AnotherBaseTest(2, 1), + mod.MyBaseTest(10, 20)] == checks - def test_run_hooks_in_run_only_test(self): - @fixtures.custom_prefix('unittests/resources/checks') - class MyTest(rfm.RunOnlyRegressionTest): - def __init__(self): - self.executable = 'echo' - self.executable_opts = ['Hello, World!'] - self.local = True - self.valid_prog_environs = ['*'] - self.valid_systems = ['*'] - self.sanity_patterns = sn.assert_found( - r'Hello, World\!', self.stdout) - @rfm.run_before('run') - def check_empty_stage(self): - # Make sure nothing has been copied to the stage directory yet - assert len(os.listdir(self.stagedir)) == 0 +def _run_sanity(test, *exec_ctx, skip_perf=False): + test.setup(*exec_ctx) + test.check_sanity() + if not skip_perf: + test.check_performance() - test = MyTest() - _run(test, self.partition, self.prgenv) - def test_multiple_hooks(self): - @fixtures.custom_prefix('unittests/resources/checks') - class MyTest(HelloTest): - def __init__(self): - super().__init__() - self.name = type(self).__name__ - self.executable = os.path.join('.', self.name) - self.var = 0 - - @rfm.run_after('setup') - def x(self): - self.var += 1 - - @rfm.run_after('setup') - def y(self): - self.var += 1 - - @rfm.run_after('setup') - def z(self): - self.var += 1 - - test = MyTest() - _run(test, self.partition, self.prgenv) - assert test.var == 3 - - def test_stacked_hooks(self): - @fixtures.custom_prefix('unittests/resources/checks') - class MyTest(HelloTest): - def __init__(self): - super().__init__() - self.name = type(self).__name__ - self.executable = os.path.join('.', self.name) - self.var = 0 - - @rfm.run_before('setup') - @rfm.run_after('setup') - @rfm.run_after('compile') - def x(self): - self.var += 1 - - test = MyTest() - _run(test, self.partition, self.prgenv) - assert test.var == 3 - - def test_inherited_hooks(self): - @fixtures.custom_prefix('unittests/resources/checks') - class BaseTest(HelloTest): - def __init__(self): - super().__init__() - self.name = type(self).__name__ - self.executable = os.path.join('.', self.name) - self.var = 0 - - @rfm.run_after('setup') - def x(self): - self.var += 1 - - class C(rfm.RegressionTest): - @rfm.run_before('run') - def y(self): - self.foo = 1 - - class DerivedTest(BaseTest, C): - @rfm.run_after('setup') - def z(self): - self.var += 1 - - class MyTest(DerivedTest): - pass - - test = MyTest() - _run(test, self.partition, self.prgenv) - assert test.var == 2 - assert test.foo == 1 - - def test_overriden_hooks(self): - @fixtures.custom_prefix('unittests/resources/checks') - class BaseTest(HelloTest): - def __init__(self): - super().__init__() - self.name = type(self).__name__ - self.executable = os.path.join('.', self.name) - self.var = 0 - self.foo = 0 - - @rfm.run_after('setup') - def x(self): - self.var += 1 - - @rfm.run_before('setup') - def y(self): - self.foo += 1 - - class DerivedTest(BaseTest): - @rfm.run_after('setup') - def x(self): - self.var += 5 - - class MyTest(DerivedTest): - @rfm.run_before('setup') - def y(self): - self.foo += 10 - - test = MyTest() - _run(test, self.partition, self.prgenv) - assert test.var == 5 - assert test.foo == 10 - - def test_require_deps(self): - import reframe.frontend.dependency as dependency - import reframe.frontend.executors as executors - - @fixtures.custom_prefix('unittests/resources/checks') - class T0(HelloTest): - def __init__(self): - super().__init__() - self.name = type(self).__name__ - self.executable = os.path.join('.', self.name) - self.x = 1 +@pytest.fixture +def dummy_gpu_exec_ctx(testsys_system): + partition = fixtures.partition_by_name('gpu') + environ = fixtures.environment_by_name('builtin-gcc', partition) + yield partition, environ - @fixtures.custom_prefix('unittests/resources/checks') - class T1(HelloTest): - def __init__(self): - super().__init__() - self.name = type(self).__name__ - self.executable = os.path.join('.', self.name) - self.depends_on('T0') - - @rfm.require_deps - def sety(self, T0): - self.y = T0().x + 1 - - @rfm.run_before('run') - @rfm.require_deps - def setz(self, T0): - self.z = T0().x + 2 - - cases = executors.generate_testcases([T0(), T1()]) - deps = dependency.build_deps(cases) - for c in dependency.toposort(deps): - _run(*c) - - for c in cases: - t = c.check - if t.name == 'T0': - assert t.x == 1 - elif t.name == 'T1': - assert t.y == 2 - assert t.z == 3 - - -class TestSyntax(unittest.TestCase): - def test_regression_test(self): - class MyTest(rfm.RegressionTest): - def __init__(self, a, b): - self.a = a - self.b = b - - test = MyTest(1, 2) - assert os.path.abspath(os.path.dirname(__file__)) == test.prefix - assert ('TestSyntax.test_regression_test..MyTest_1_2' == - test.name) - - def test_regression_test_strange_names(self): - class C: - def __init__(self, a): - self.a = a - - def __repr__(self): - return 'C(%s)' % self.a - - class MyTest(rfm.RegressionTest): - def __init__(self, a, b): - self.a = a - self.b = b - - test = MyTest('(a*b+c)/12', C(33)) - assert ('TestSyntax.test_regression_test_strange_names.' - '.MyTest__a_b_c__12_C_33_' == test.name) - - def test_user_inheritance(self): - class MyBaseTest(rfm.RegressionTest): - def __init__(self, a, b): - self.a = a - self.b = b - - class MyTest(MyBaseTest): - def __init__(self): - super().__init__(1, 2) - - test = MyTest() - assert 'TestSyntax.test_user_inheritance..MyTest' == test.name - - def test_runonly_test(self): - class MyTest(rfm.RunOnlyRegressionTest): - def __init__(self, a, b): - self.a = a - self.b = b - - test = MyTest(1, 2) - assert os.path.abspath(os.path.dirname(__file__)) == test.prefix - assert 'TestSyntax.test_runonly_test..MyTest_1_2' == test.name - - def test_compileonly_test(self): - class MyTest(rfm.CompileOnlyRegressionTest): - def __init__(self, a, b): - self.a = a - self.b = b - - test = MyTest(1, 2) - assert os.path.abspath(os.path.dirname(__file__)) == test.prefix - assert ('TestSyntax.test_compileonly_test..MyTest_1_2' == - test.name) - - def test_registration(self): - import sys - import unittests.resources.checks_unlisted.good as mod - checks = mod._rfm_gettests() - assert 13 == len(checks) - assert [mod.MyBaseTest(0, 0), - mod.MyBaseTest(0, 1), - mod.MyBaseTest(1, 0), - mod.MyBaseTest(1, 1), - mod.MyBaseTest(2, 0), - mod.MyBaseTest(2, 1), - mod.AnotherBaseTest(0, 0), - mod.AnotherBaseTest(0, 1), - mod.AnotherBaseTest(1, 0), - mod.AnotherBaseTest(1, 1), - mod.AnotherBaseTest(2, 0), - mod.AnotherBaseTest(2, 1), - mod.MyBaseTest(10, 20)] == checks - - -class TestSanityPatterns(unittest.TestCase): - @rt.switch_runtime(fixtures.TEST_SITE_CONFIG, 'testsys') - def setUp(self): - # Set up the test runtime - self.resourcesdir = tempfile.mkdtemp(dir='unittests') - rt.runtime().resources.prefix = self.resourcesdir - - # Set up regression test - @fixtures.custom_prefix('unittests/resources/checks') - class MyTest(rfm.RegressionTest): - pass - - self.partition = rt.runtime().system.partition('gpu') - self.prgenv = self.partition.environment('builtin-gcc') - - self.test = MyTest() - self.test.setup(self.partition, self.prgenv) - self.test.reference = { - 'testsys': { - 'value1': (1.4, -0.1, 0.1, None), - 'value2': (1.7, -0.1, 0.1, None), - }, - 'testsys:gpu': { - 'value3': (3.1, -0.1, 0.1, None), - } - } - self.perf_file = tempfile.NamedTemporaryFile(mode='wt', delete=False) - self.output_file = tempfile.NamedTemporaryFile(mode='wt', delete=False) - self.test.perf_patterns = { - 'value1': sn.extractsingle(r'performance1 = (\S+)', - self.perf_file.name, 1, float), - 'value2': sn.extractsingle(r'performance2 = (\S+)', - self.perf_file.name, 1, float), - 'value3': sn.extractsingle(r'performance3 = (\S+)', - self.perf_file.name, 1, float) - } - self.test.sanity_patterns = sn.assert_found(r'result = success', - self.output_file.name) - - def tearDown(self): - self.perf_file.close() - self.output_file.close() - os.remove(self.perf_file.name) - os.remove(self.output_file.name) - os_ext.rmtree(self.resourcesdir) - - def write_performance_output(self, fp=None, **kwargs): - if not fp: - fp = self.perf_file - - for k, v in kwargs.items(): - fp.write('%s = %s\n' % (k, v)) - - fp.close() - - def test_success(self): - self.write_performance_output(performance1=1.3, - performance2=1.8, - performance3=3.3) - self.output_file.write('result = success\n') - self.output_file.close() - self.test.check_sanity() - self.test.check_performance() - - def test_sanity_failure(self): - self.output_file.write('result = failure\n') - self.output_file.close() - with pytest.raises(SanityError): - self.test.check_sanity() - - def test_sanity_failure_noassert(self): - self.test.sanity_patterns = sn.findall(r'result = success', - self.output_file.name) - self.output_file.write('result = failure\n') - self.output_file.close() - with pytest.raises(SanityError): - self.test.check_sanity() - - def test_sanity_multiple_patterns(self): - self.output_file.write('result1 = success\n') - self.output_file.write('result2 = success\n') - self.output_file.close() - - # Simulate a pure sanity test; invalidate the reference values - self.test.reference = {} - self.test.sanity_patterns = sn.assert_eq( - sn.count(sn.findall(r'result\d = success', self.output_file.name)), - 2) - self.test.check_sanity() - - # Require more patterns to be present - self.test.sanity_patterns = sn.assert_eq( - sn.count(sn.findall(r'result\d = success', self.output_file.name)), - 3) - with pytest.raises(SanityError): - self.test.check_sanity() - - def test_sanity_multiple_files(self): - files = [tempfile.NamedTemporaryFile(mode='wt', prefix='regtmp', - dir=self.test.stagedir, - delete=False) - for i in range(2)] - - for f in files: - f.write('result = success\n') - f.close() - - self.test.sanity_patterns = sn.all([ - sn.assert_found(r'result = success', files[0].name), - sn.assert_found(r'result = success', files[1].name) - ]) - self.test.check_sanity() - for f in files: - os.remove(f.name) - - def test_performance_failure(self): - self.write_performance_output(performance1=1.0, - performance2=1.8, - performance3=3.3) - self.output_file.write('result = success\n') - self.output_file.close() - self.test.check_sanity() - with pytest.raises(PerformanceError): - self.test.check_performance() - - def test_performance_no_units(self): - with pytest.raises(TypeError): - self.test.reference = { - 'testsys': { - 'value1': (1.4, -0.1, 0.1), - } - } +@pytest.fixture +def perf_file(tmp_path): + yield tmp_path / 'perf.out' - def test_unknown_tag(self): - self.test.reference = { - 'testsys': { - 'value1': (1.4, -0.1, 0.1, None), - 'value2': (1.7, -0.1, 0.1, None), - 'foo': (3.1, -0.1, 0.1, None), - } - } - self.write_performance_output(performance1=1.3, - performance2=1.8, - performance3=3.3) - with pytest.raises(SanityError): - self.test.check_performance() - - def test_unknown_system(self): - self.write_performance_output(performance1=1.3, - performance2=1.8, - performance3=3.3) - self.test.reference = { - 'testsys:login': { - 'value1': (1.4, -0.1, 0.1, None), - 'value3': (3.1, -0.1, 0.1, None), - }, - 'testsys:login2': { - 'value2': (1.7, -0.1, 0.1, None) +@pytest.fixture +def sanity_file(tmp_path): + yield tmp_path / 'sanity.out' + + +@pytest.fixture +def dummytest(testsys_system, perf_file, sanity_file): + class MyTest(rfm.RunOnlyRegressionTest): + def __init__(self): + self.perf_file = perf_file + self.sourcesdir = None + self.reference = { + 'testsys': { + 'value1': (1.4, -0.1, 0.1, None), + 'value2': (1.7, -0.1, 0.1, None), + }, + 'testsys:gpu': { + 'value3': (3.1, -0.1, 0.1, None), + } } - } - self.test.check_performance() - - def test_empty_reference(self): - self.write_performance_output(performance1=1.3, - performance2=1.8, - performance3=3.3) - self.test.reference = {} - self.test.check_performance() - - def test_default_reference(self): - self.write_performance_output(performance1=1.3, - performance2=1.8, - performance3=3.3) - self.test.reference = { - '*': { - 'value1': (1.4, -0.1, 0.1, None), - 'value2': (1.7, -0.1, 0.1, None), - 'value3': (3.1, -0.1, 0.1, None), + self.perf_patterns = { + 'value1': sn.extractsingle( + r'perf1 = (\S+)', perf_file, 1, float + ), + 'value2': sn.extractsingle( + r'perf2 = (\S+)', perf_file, 1, float + ), + 'value3': sn.extractsingle( + r'perf3 = (\S+)', perf_file, 1, float + ) } + self.sanity_patterns = sn.assert_found( + r'result = success', sanity_file + ) + + yield MyTest() + + +def test_sanity_success(dummytest, sanity_file, perf_file, dummy_gpu_exec_ctx): + sanity_file.write_text('result = success\n') + perf_file.write_text('perf1 = 1.3\n' + 'perf2 = 1.8\n' + 'perf3 = 3.3\n') + _run_sanity(dummytest, *dummy_gpu_exec_ctx) + + +def test_sanity_failure(dummytest, sanity_file, dummy_gpu_exec_ctx): + sanity_file.write_text('result = failure\n') + with pytest.raises(SanityError): + _run_sanity(dummytest, *dummy_gpu_exec_ctx, skip_perf=True) + + +def test_sanity_failure_noassert(dummytest, sanity_file, dummy_gpu_exec_ctx): + dummytest.sanity_patterns = sn.findall(r'result = success', sanity_file) + sanity_file.write_text('result = failure\n') + with pytest.raises(SanityError): + _run_sanity(dummytest, *dummy_gpu_exec_ctx, skip_perf=True) + + +def test_sanity_multiple_patterns(dummytest, sanity_file, dummy_gpu_exec_ctx): + sanity_file.write_text('result1 = success\n' + 'result2 = success\n') + + # Simulate a pure sanity test; reset the perf_patterns + dummytest.perf_patterns = None + dummytest.sanity_patterns = sn.assert_eq( + sn.count(sn.findall(r'result\d = success', sanity_file)), 2 + ) + _run_sanity(dummytest, *dummy_gpu_exec_ctx, skip_perf=True) + + # Require more patterns to be present + dummytest.sanity_patterns = sn.assert_eq( + sn.count(sn.findall(r'result\d = success', sanity_file)), 3 + ) + with pytest.raises(SanityError): + _run_sanity(dummytest, *dummy_gpu_exec_ctx, skip_perf=True) + + +def test_sanity_multiple_files(dummytest, tmp_path, dummy_gpu_exec_ctx): + file0 = tmp_path / 'out1.txt' + file1 = tmp_path / 'out2.txt' + file0.write_text('result = success\n') + file1.write_text('result = success\n') + dummytest.sanity_patterns = sn.all([ + sn.assert_found(r'result = success', file0), + sn.assert_found(r'result = success', file1) + ]) + _run_sanity(dummytest, *dummy_gpu_exec_ctx, skip_perf=True) + + +def test_performance_failure(dummytest, sanity_file, + perf_file, dummy_gpu_exec_ctx): + sanity_file.write_text('result = success\n') + perf_file.write_text('perf1 = 1.0\n' + 'perf2 = 1.8\n' + 'perf3 = 3.3\n') + with pytest.raises(PerformanceError): + _run_sanity(dummytest, *dummy_gpu_exec_ctx) + + +def test_reference_unknown_tag(dummytest, sanity_file, + perf_file, dummy_gpu_exec_ctx): + sanity_file.write_text('result = success\n') + perf_file.write_text('perf1 = 1.3\n' + 'perf2 = 1.8\n' + 'perf3 = 3.3\n') + dummytest.reference = { + 'testsys': { + 'value1': (1.4, -0.1, 0.1, None), + 'value2': (1.7, -0.1, 0.1, None), + 'foo': (3.1, -0.1, 0.1, None), } - - self.test.check_performance() - - def test_tag_resolution(self): - self.write_performance_output(performance1=1.3, - performance2=1.8, - performance3=3.3) - self.test.reference = { - 'testsys': { - 'value1': (1.4, -0.1, 0.1, None), - 'value2': (1.7, -0.1, 0.1, None), - }, - '*': { - 'value3': (3.1, -0.1, 0.1, None), - } + } + with pytest.raises(SanityError): + _run_sanity(dummytest, *dummy_gpu_exec_ctx) + + +def test_reference_unknown_system(dummytest, sanity_file, + perf_file, dummy_gpu_exec_ctx): + sanity_file.write_text('result = success\n') + perf_file.write_text('perf1 = 1.3\n' + 'perf2 = 1.8\n' + 'perf3 = 3.3\n') + dummytest.reference = { + 'testsys:login': { + 'value1': (1.4, -0.1, 0.1, None), + 'value3': (3.1, -0.1, 0.1, None), + }, + 'testsys:login2': { + 'value2': (1.7, -0.1, 0.1, None) } - self.test.check_performance() - - def test_invalid_perf_value(self): - self.test.perf_patterns = { - 'value1': sn.extractsingle(r'performance1 = (\S+)', - self.perf_file.name, 1, float), - 'value2': sn.extractsingle(r'performance2 = (\S+)', - self.perf_file.name, 1, str), - 'value3': sn.extractsingle(r'performance3 = (\S+)', - self.perf_file.name, 1, float) + } + _run_sanity(dummytest, *dummy_gpu_exec_ctx) + + +def test_reference_empty(dummytest, sanity_file, + perf_file, dummy_gpu_exec_ctx): + sanity_file.write_text('result = success\n') + perf_file.write_text('perf1 = 1.3\n' + 'perf2 = 1.8\n' + 'perf3 = 3.3\n') + dummytest.reference = {} + _run_sanity(dummytest, *dummy_gpu_exec_ctx) + + +def test_reference_default(dummytest, sanity_file, + perf_file, dummy_gpu_exec_ctx): + sanity_file.write_text('result = success\n') + perf_file.write_text('perf1 = 1.3\n' + 'perf2 = 1.8\n' + 'perf3 = 3.3\n') + dummytest.reference = { + '*': { + 'value1': (1.4, -0.1, 0.1, None), + 'value2': (1.7, -0.1, 0.1, None), + 'value3': (3.1, -0.1, 0.1, None), } - self.write_performance_output(performance1=1.3, - performance2='foo', - performance3=3.3) - with pytest.raises(SanityError, match='not a number'): - self.test.check_performance() - - def test_perf_var_evaluation(self): - # All performance values must be evaluated, despite the first one - # failing To test this, we need an extract function that will have a - # side effect when evaluated, whose result we will check after calling - # `check_performance()`. - logfile = 'perf.log' - - @sn.sanity_function - def extract_perf(patt, tag): - val = sn.evaluate( - sn.extractsingle(patt, self.perf_file.name, tag, float)) - - with open('perf.log', 'a') as fp: - fp.write('%s=%s' % (tag, val)) - - return val - - self.test.perf_patterns = { - 'value1': extract_perf(r'performance1 = (?P\S+)', 'v1'), - 'value2': extract_perf(r'performance2 = (?P\S+)', 'v2'), - 'value3': extract_perf(r'performance3 = (?P\S+)', 'v3') + } + _run_sanity(dummytest, *dummy_gpu_exec_ctx) + + +def test_reference_tag_resolution(dummytest, sanity_file, + perf_file, dummy_gpu_exec_ctx): + sanity_file.write_text('result = success\n') + perf_file.write_text('perf1 = 1.3\n' + 'perf2 = 1.8\n' + 'perf3 = 3.3\n') + dummytest.reference = { + 'testsys': { + 'value1': (1.4, -0.1, 0.1, None), + 'value2': (1.7, -0.1, 0.1, None), + }, + '*': { + 'value3': (3.1, -0.1, 0.1, None), } - self.write_performance_output(performance1=1.0, - performance2=1.8, - performance3=3.3) - with pytest.raises(PerformanceError) as cm: - self.test.check_performance() - - logfile = os.path.join(self.test.stagedir, logfile) - with open(logfile) as fp: - log_output = fp.read() - - assert 'v1' in log_output - assert 'v2' in log_output - assert 'v3' in log_output - - -class TestRegressionTestWithContainer(unittest.TestCase): - def temp_prefix(self): - # Set runtime prefix - rt.runtime().resources.prefix = tempfile.mkdtemp(dir='unittests') - - def create_test(self, platform, image): - @fixtures.custom_prefix('unittests/resources/checks') + } + _run_sanity(dummytest, *dummy_gpu_exec_ctx) + + +def test_performance_invalid_value(dummytest, sanity_file, + perf_file, dummy_gpu_exec_ctx): + sanity_file.write_text('result = success\n') + perf_file.write_text('perf1 = 1.3\n' + 'perf2 = foo\n' + 'perf3 = 3.3\n') + dummytest.perf_patterns = { + 'value1': sn.extractsingle(r'perf1 = (\S+)', perf_file, 1, float), + 'value2': sn.extractsingle(r'perf2 = (\S+)', perf_file, 1, str), + 'value3': sn.extractsingle(r'perf3 = (\S+)', perf_file, 1, float) + } + with pytest.raises(SanityError, match='not a number'): + _run_sanity(dummytest, *dummy_gpu_exec_ctx) + + +def test_performance_var_evaluation(dummytest, sanity_file, + perf_file, dummy_gpu_exec_ctx): + # All performance values must be evaluated, despite the first one + # failing To test this, we need an extract function that will have a + # side effect when evaluated, whose result we will check after calling + # `check_performance()`. + logfile = 'perf.log' + + @sn.sanity_function + def extract_perf(patt, tag): + val = sn.evaluate( + sn.extractsingle(patt, perf_file, tag, float) + ) + with open('perf.log', 'a') as fp: + fp.write('%s=%s' % (tag, val)) + + return val + + sanity_file.write_text('result = success\n') + perf_file.write_text('perf1 = 1.0\n' + 'perf2 = 1.8\n' + 'perf3 = 3.3\n') + dummytest.perf_patterns = { + 'value1': extract_perf(r'perf1 = (?P\S+)', 'v1'), + 'value2': extract_perf(r'perf2 = (?P\S+)', 'v2'), + 'value3': extract_perf(r'perf3 = (?P\S+)', 'v3') + } + with pytest.raises(PerformanceError) as cm: + _run_sanity(dummytest, *dummy_gpu_exec_ctx) + + logfile = os.path.join(dummytest.stagedir, logfile) + with open(logfile) as fp: + log_output = fp.read() + + assert 'v1' in log_output + assert 'v2' in log_output + assert 'v3' in log_output + + +@pytest.fixture +def container_test(tmp_path): + def _container_test(platform, image): + @fixtures.custom_prefix(tmp_path) class ContainerTest(rfm.RunOnlyRegressionTest): - def __init__(self, platform): + def __init__(self): + self.name = 'container_test' self.valid_prog_environs = ['*'] self.valid_systems = ['*'] self.container_platform = platform @@ -1062,82 +1026,68 @@ def __init__(self, platform): 'pwd', 'ls', 'cat /etc/os-release' ] self.container_platform.workdir = '/workdir' + self.pre_run = ['touch foo'] self.sanity_patterns = sn.all([ sn.assert_found( r'^' + self.container_platform.workdir, self.stdout), - sn.assert_found(r'^hello.c', self.stdout), + sn.assert_found(r'^foo', self.stdout), sn.assert_found( r'18\.04\.\d+ LTS \(Bionic Beaver\)', self.stdout), ]) - test = ContainerTest(platform) - return test + return ContainerTest() - def _skip_if_not_configured(self, partition, platform): - if platform not in partition.container_environs.keys(): - pytest.skip('%s is not configured on the system' % platform) + yield _container_test + + +def _cray_cle_version(): + completed = os_ext.run_command('cat /etc/opt/cray/release/cle-release') + matched = re.match(r'^RELEASE=(\S+)', completed.stdout) + if matched is None: + return None + + return matched.group(1) + + +def test_with_singularity(container_test, container_remote_exec_ctx): + cle_version = _cray_cle_version() + if cle_version is not None and cle_version.startswith('6.0'): + pytest.skip('test not supported on Cray CLE6') + + _run(container_test('Singularity', 'docker://ubuntu:18.04'), + *container_remote_exec_ctx('Singularity')) + + +def test_with_shifter(container_test, container_remote_exec_ctx): + _run(container_test('Shifter', 'ubuntu:18.04'), + *container_remote_exec_ctx('Shifter')) + + +def test_with_sarus(container_test, container_remote_exec_ctx): + _run(container_test('Sarus', 'ubuntu:18.04'), + *container_remote_exec_ctx('Sarus')) + + +def test_with_docker(container_test, container_local_exec_ctx): + _run(container_test('Docker', 'ubuntu:18.04'), + *container_local_exec_ctx('Docker')) + + +def test_unknown_container_platform(container_test, local_exec_ctx): + with pytest.raises(ValueError): + _run(container_test('foo', 'ubuntu:18.04'), *local_exec_ctx) + + +def test_not_configured_container_platform(container_test, local_exec_ctx): + partition, environ = local_exec_ctx + platform = None + for cp in ['Docker', 'Singularity', 'Sarus', 'ShifterNG']: + if cp not in partition.container_environs.keys(): + platform = cp + break + + if platform is None: + pytest.skip('cannot find a supported platform that is not configured') - @fixtures.switch_to_user_runtime - def test_singularity(self): - cle_version = _cray_cle_version() - if cle_version is not None and cle_version.startswith('6.0'): - pytest.skip('test not supported on Cray CLE6') - - partition, environ = _setup_remote_execution() - self._skip_if_not_configured(partition, 'Singularity') - with tempfile.TemporaryDirectory(dir='unittests') as dirname: - rt.runtime().resources.prefix = dirname - _run(self.create_test('Singularity', 'docker://ubuntu:18.04'), - partition, environ) - - @fixtures.switch_to_user_runtime - def test_docker(self): - partition, environ = _setup_remote_execution('local') - self._skip_if_not_configured(partition, 'Docker') - with tempfile.TemporaryDirectory(dir='unittests') as dirname: - rt.runtime().resources.prefix = dirname - _run(self.create_test('Docker', 'ubuntu:18.04'), - partition, environ) - - @fixtures.switch_to_user_runtime - def test_shifter(self): - partition, environ = _setup_remote_execution() - self._skip_if_not_configured(partition, 'ShifterNG') - with tempfile.TemporaryDirectory(dir='unittests') as dirname: - rt.runtime().resources.prefix = dirname - _run(self.create_test('ShifterNG', 'ubuntu:18.04'), - partition, environ) - - @fixtures.switch_to_user_runtime - def test_sarus(self): - partition, environ = _setup_remote_execution() - self._skip_if_not_configured(partition, 'Sarus') - with tempfile.TemporaryDirectory(dir='unittests') as dirname: - rt.runtime().resources.prefix = dirname - _run(self.create_test('Sarus', 'ubuntu:18.04'), - partition, environ) - - def test_unknown_platform(self): - partition, environ = _setup_local_execution() - with pytest.raises(ValueError): - with tempfile.TemporaryDirectory(dir='unittests') as dirname: - rt.runtime().resources.prefix = dirname - _run(self.create_test('foo', 'ubuntu:18.04'), - partition, environ) - - def test_not_configured_platform(self): - partition, environ = _setup_local_execution() - platform = None - for cp in ['Docker', 'Singularity', 'Sarus', 'ShifterNG']: - if cp not in partition.container_environs.keys(): - platform = cp - break - - if platform is None: - pytest.skip('cannot find a not configured supported platform') - - with pytest.raises(PipelineError): - with tempfile.TemporaryDirectory(dir='unittests') as dirname: - rt.runtime().resources.prefix = dirname - _run(self.create_test(platform, 'ubuntu:18.04'), - partition, environ) + with pytest.raises(PipelineError): + _run(container_test(platform, 'ubuntu:18.04'), *local_exec_ctx) diff --git a/unittests/test_policies.py b/unittests/test_policies.py index 7b46744f79..9093183a3f 100644 --- a/unittests/test_policies.py +++ b/unittests/test_policies.py @@ -3,27 +3,19 @@ # # SPDX-License-Identifier: BSD-3-Clause -import collections -import itertools import os import pytest -import time -import tempfile -import unittest -import reframe as rfm import reframe.core.runtime as rt import reframe.frontend.dependency as dependency import reframe.frontend.executors as executors import reframe.frontend.executors.policies as policies -import reframe.utility as util import reframe.utility.os_ext as os_ext -from reframe.core.environments import Environment -from reframe.core.exceptions import ( - DependencyError, JobNotStartedError, - ReframeForceExitError, TaskDependencyError -) +from reframe.core.exceptions import (JobNotStartedError, + ReframeForceExitError, + TaskDependencyError) from reframe.frontend.loader import RegressionCheckLoader + import unittests.fixtures as fixtures from unittests.resources.checks.hellocheck import HelloTest from unittests.resources.checks.frontend_checks import ( @@ -31,6 +23,7 @@ BadSetupCheckEarly, KeyboardInterruptCheck, RetriesCheck, + SelfKillCheck, SleepCheck, SleepCheckPollFail, SleepCheckPollFailLate, @@ -38,246 +31,286 @@ ) -class TestSerialExecutionPolicy(unittest.TestCase): - def setUp(self): - self.loader = RegressionCheckLoader(['unittests/resources/checks'], - ignore_conflicts=True) +@pytest.fixture +def temp_runtime(tmp_path): + def _temp_runtime(site_config, system=None, options={}): + options.update({'systems/prefix': str(tmp_path)}) + with rt.temp_runtime(site_config, system, options): + yield rt.runtime + + yield _temp_runtime + + +@pytest.fixture +def make_loader(): + def _make_loader(check_search_path): + return RegressionCheckLoader(check_search_path, + ignore_conflicts=True) - # Setup the runner - self.runner = executors.Runner(policies.SerialExecutionPolicy()) - self.checks = self.loader.load_all() + return _make_loader - # Set runtime prefix - rt.runtime().resources.prefix = tempfile.mkdtemp(dir='unittests') - # Reset current_run - rt.runtime()._current_run = 0 +@pytest.fixture +def common_exec_ctx(temp_runtime): + yield from temp_runtime(fixtures.TEST_CONFIG_FILE, 'generic') - def tearDown(self): - os_ext.rmtree(rt.runtime().resources.prefix) - def runall(self, checks, sort=False, *args, **kwargs): +@pytest.fixture(params=[policies.SerialExecutionPolicy, + policies.AsynchronousExecutionPolicy]) +def make_runner(request): + def _make_runner(*args, **kwargs): + return executors.Runner(request.param(), *args, **kwargs) + + return _make_runner + + +@pytest.fixture +def make_cases(make_loader): + def _make_cases(checks=None, sort=False, *args, **kwargs): + if checks is None: + checks = make_loader(['unittests/resources/checks']).load_all() + cases = executors.generate_testcases(checks, *args, **kwargs) if sort: depgraph = dependency.build_deps(cases) dependency.validate_deps(depgraph) cases = dependency.toposort(depgraph) - self.runner.runall(cases) - - def assertRunall(self): - # Make sure that all cases finished or failed - for t in self.runner.stats.tasks(): - assert t.succeeded or t.failed - - def _num_failures_stage(self, stage): - stats = self.runner.stats - return len([t for t in stats.failures() if t.failed_stage == stage]) - - def assert_all_dead(self): - stats = self.runner.stats - for t in self.runner.stats.tasks(): - try: - finished = t.check.poll() - except JobNotStartedError: - finished = True - - assert finished - - def test_runall(self): - self.runall(self.checks) - - stats = self.runner.stats - assert 8 == stats.num_cases() - self.assertRunall() - assert 5 == len(stats.failures()) - assert 2 == self._num_failures_stage('setup') - assert 1 == self._num_failures_stage('sanity') - assert 1 == self._num_failures_stage('performance') - assert 1 == self._num_failures_stage('cleanup') - - def test_runall_skip_system_check(self): - self.runall(self.checks, skip_system_check=True) - - stats = self.runner.stats - assert 9 == stats.num_cases() - self.assertRunall() - assert 5 == len(stats.failures()) - assert 2 == self._num_failures_stage('setup') - assert 1 == self._num_failures_stage('sanity') - assert 1 == self._num_failures_stage('performance') - assert 1 == self._num_failures_stage('cleanup') - - def test_runall_skip_prgenv_check(self): - self.runall(self.checks, skip_environ_check=True) - - stats = self.runner.stats - assert 9 == stats.num_cases() - self.assertRunall() - assert 5 == len(stats.failures()) - assert 2 == self._num_failures_stage('setup') - assert 1 == self._num_failures_stage('sanity') - assert 1 == self._num_failures_stage('performance') - assert 1 == self._num_failures_stage('cleanup') - - def test_runall_skip_sanity_check(self): - self.runner.policy.skip_sanity_check = True - self.runall(self.checks) - - stats = self.runner.stats - assert 8 == stats.num_cases() - self.assertRunall() - assert 4 == len(stats.failures()) - assert 2 == self._num_failures_stage('setup') - assert 0 == self._num_failures_stage('sanity') - assert 1 == self._num_failures_stage('performance') - assert 1 == self._num_failures_stage('cleanup') - - def test_runall_skip_performance_check(self): - self.runner.policy.skip_performance_check = True - self.runall(self.checks) - - stats = self.runner.stats - assert 8 == stats.num_cases() - self.assertRunall() - assert 4 == len(stats.failures()) - assert 2 == self._num_failures_stage('setup') - assert 1 == self._num_failures_stage('sanity') - assert 0 == self._num_failures_stage('performance') - assert 1 == self._num_failures_stage('cleanup') - - def test_strict_performance_check(self): - self.runner.policy.strict_check = True - self.runall(self.checks) - - stats = self.runner.stats - assert 8 == stats.num_cases() - self.assertRunall() - assert 6 == len(stats.failures()) - assert 2 == self._num_failures_stage('setup') - assert 1 == self._num_failures_stage('sanity') - assert 2 == self._num_failures_stage('performance') - assert 1 == self._num_failures_stage('cleanup') - - def test_force_local_execution(self): - self.runner.policy.force_local = True - self.runall([HelloTest()]) - self.assertRunall() - stats = self.runner.stats - for t in stats.tasks(): - assert t.check.local - - def test_kbd_interrupt_within_test(self): - check = KeyboardInterruptCheck() - with pytest.raises(KeyboardInterrupt): - self.runall([check]) - - stats = self.runner.stats - assert 1 == len(stats.failures()) - self.assert_all_dead() - - def test_system_exit_within_test(self): - check = SystemExitCheck() - - # This should not raise and should not exit - self.runall([check]) - stats = self.runner.stats - assert 1 == len(stats.failures()) - - def test_retries_bad_check(self): - max_retries = 2 - checks = [BadSetupCheck(), BadSetupCheckEarly()] - self.runner._max_retries = max_retries - self.runall(checks) - - # Ensure that the test was retried #max_retries times and failed. - assert 2 == self.runner.stats.num_cases() - self.assertRunall() - assert max_retries == rt.runtime().current_run - assert 2 == len(self.runner.stats.failures()) - - # Ensure that the report does not raise any exception. - self.runner.stats.retry_report() - - def test_retries_good_check(self): - max_retries = 2 - checks = [HelloTest()] - self.runner._max_retries = max_retries - self.runall(checks) - - # Ensure that the test passed without retries. - assert 1 == self.runner.stats.num_cases() - self.assertRunall() - assert 0 == rt.runtime().current_run - assert 0 == len(self.runner.stats.failures()) - - def test_pass_in_retries(self): - max_retries = 3 - run_to_pass = 2 - # Create a file containing the current_run; Run 0 will set it to 0, - # run 1 to 1 and so on. - with tempfile.NamedTemporaryFile(mode='wt', delete=False) as fp: - fp.write('0\n') - - checks = [RetriesCheck(run_to_pass, fp.name)] - self.runner._max_retries = max_retries - self.runall(checks) - - # Ensure that the test passed after retries in run #run_to_pass. - assert 1 == self.runner.stats.num_cases() - self.assertRunall() - assert 1 == len(self.runner.stats.failures(run=0)) - assert run_to_pass == rt.runtime().current_run - assert 0 == len(self.runner.stats.failures()) - os.remove(fp.name) - - def test_dependencies(self): - self.loader = RegressionCheckLoader( - ['unittests/resources/checks_unlisted/deps_complex.py'] - ) - - # Setup the runner - self.checks = self.loader.load_all() - self.runall(self.checks, sort=True) - - self.assertRunall() - stats = self.runner.stats - assert stats.num_cases(0) == 10 - assert len(stats.failures()) == 4 - for tf in stats.failures(): - check = tf.testcase.check - _, exc_value, _ = tf.exc_info - if check.name == 'T7' or check.name == 'T9': - assert isinstance(exc_value, TaskDependencyError) - - # Check that cleanup is executed properly for successful tests as well - for t in stats.tasks(): - check = t.testcase.check - if t.failed: - continue - - if t.ref_count == 0: - assert os.path.exists(os.path.join(check.outputdir, 'out.txt')) - - def test_sigterm(self): - self.loader = RegressionCheckLoader( - ['unittests/resources/checks_unlisted/selfkill.py'] - ) - checks = self.loader.load_all() - with pytest.raises(ReframeForceExitError, - match='received TERM signal'): - self.runall(checks) - - self.assert_all_dead() - assert self.runner.stats.num_cases() == 1 - assert len(self.runner.stats.failures()) == 1 - - def test_dependencies_with_retries(self): - self.runner._max_retries = 2 - self.test_dependencies() - - -class TaskEventMonitor(executors.TaskEventListener): + return cases + + return _make_cases + + +def assert_runall(runner): + # Make sure that all cases finished or failed + for t in runner.stats.tasks(): + assert t.succeeded or t.failed + + +def assert_all_dead(runner): + stats = runner.stats + for t in runner.stats.tasks(): + try: + finished = t.check.poll() + except JobNotStartedError: + finished = True + + assert finished + + +def num_failures_stage(runner, stage): + stats = runner.stats + return len([t for t in stats.failures() if t.failed_stage == stage]) + + +def test_runall(make_runner, make_cases, common_exec_ctx): + runner = make_runner() + runner.runall(make_cases()) + stats = runner.stats + assert 8 == stats.num_cases() + assert_runall(runner) + assert 5 == len(stats.failures()) + assert 2 == num_failures_stage(runner, 'setup') + assert 1 == num_failures_stage(runner, 'sanity') + assert 1 == num_failures_stage(runner, 'performance') + assert 1 == num_failures_stage(runner, 'cleanup') + + +def test_runall_skip_system_check(make_runner, make_cases, common_exec_ctx): + runner = make_runner() + runner.runall(make_cases(skip_system_check=True)) + stats = runner.stats + assert 9 == stats.num_cases() + assert_runall(runner) + assert 5 == len(stats.failures()) + assert 2 == num_failures_stage(runner, 'setup') + assert 1 == num_failures_stage(runner, 'sanity') + assert 1 == num_failures_stage(runner, 'performance') + assert 1 == num_failures_stage(runner, 'cleanup') + + +def test_runall_skip_prgenv_check(make_runner, make_cases, common_exec_ctx): + runner = make_runner() + runner.runall(make_cases(skip_environ_check=True)) + stats = runner.stats + assert 9 == stats.num_cases() + assert_runall(runner) + assert 5 == len(stats.failures()) + assert 2 == num_failures_stage(runner, 'setup') + assert 1 == num_failures_stage(runner, 'sanity') + assert 1 == num_failures_stage(runner, 'performance') + assert 1 == num_failures_stage(runner, 'cleanup') + + +def test_runall_skip_sanity_check(make_runner, make_cases, common_exec_ctx): + runner = make_runner() + runner.policy.skip_sanity_check = True + runner.runall(make_cases()) + stats = runner.stats + assert 8 == stats.num_cases() + assert_runall(runner) + assert 4 == len(stats.failures()) + assert 2 == num_failures_stage(runner, 'setup') + assert 0 == num_failures_stage(runner, 'sanity') + assert 1 == num_failures_stage(runner, 'performance') + assert 1 == num_failures_stage(runner, 'cleanup') + + +def test_runall_skip_performance_check(make_runner, make_cases, + common_exec_ctx): + runner = make_runner() + runner.policy.skip_performance_check = True + runner.runall(make_cases()) + stats = runner.stats + assert 8 == stats.num_cases() + assert_runall(runner) + assert 4 == len(stats.failures()) + assert 2 == num_failures_stage(runner, 'setup') + assert 1 == num_failures_stage(runner, 'sanity') + assert 0 == num_failures_stage(runner, 'performance') + assert 1 == num_failures_stage(runner, 'cleanup') + + +def test_strict_performance_check(make_runner, make_cases, common_exec_ctx): + runner = make_runner() + runner.policy.strict_check = True + runner.runall(make_cases()) + stats = runner.stats + assert 8 == stats.num_cases() + assert_runall(runner) + assert 6 == len(stats.failures()) + assert 2 == num_failures_stage(runner, 'setup') + assert 1 == num_failures_stage(runner, 'sanity') + assert 2 == num_failures_stage(runner, 'performance') + assert 1 == num_failures_stage(runner, 'cleanup') + + +def test_force_local_execution(make_runner, make_cases, common_exec_ctx): + runner = make_runner() + runner.policy.force_local = True + runner.runall(make_cases([HelloTest()])) + assert_runall(runner) + stats = runner.stats + for t in stats.tasks(): + assert t.check.local + + +def test_kbd_interrupt_within_test(make_runner, make_cases, common_exec_ctx): + runner = make_runner() + check = KeyboardInterruptCheck() + with pytest.raises(KeyboardInterrupt): + runner.runall(make_cases([KeyboardInterruptCheck()])) + + stats = runner.stats + assert 1 == len(stats.failures()) + assert_all_dead(runner) + + +def test_system_exit_within_test(make_runner, make_cases, common_exec_ctx): + # This should not raise and should not exit + runner = make_runner() + runner.runall(make_cases([SystemExitCheck()])) + stats = runner.stats + assert 1 == len(stats.failures()) + + +def test_retries_bad_check(make_runner, make_cases, common_exec_ctx): + runner = make_runner(max_retries=2) + runner.runall(make_cases([BadSetupCheck(), BadSetupCheckEarly()])) + + # Ensure that the test was retried #max_retries times and failed + assert 2 == runner.stats.num_cases() + assert_runall(runner) + assert runner.max_retries == rt.runtime().current_run + assert 2 == len(runner.stats.failures()) + + # Ensure that the report does not raise any exception + runner.stats.retry_report() + + +def test_retries_good_check(make_runner, make_cases, common_exec_ctx): + runner = make_runner(max_retries=2) + runner.runall(make_cases([HelloTest()])) + + # Ensure that the test passed without retries. + assert 1 == runner.stats.num_cases() + assert_runall(runner) + assert 0 == rt.runtime().current_run + assert 0 == len(runner.stats.failures()) + + +def test_pass_in_retries(make_runner, make_cases, tmp_path, common_exec_ctx): + tmpfile = tmp_path / 'out.txt' + tmpfile.write_text('0\n') + runner = make_runner(max_retries=3) + pass_run_no = 2 + runner.runall(make_cases([RetriesCheck(pass_run_no, tmpfile)])) + + # Ensure that the test passed after retries in run `pass_run_no` + assert 1 == runner.stats.num_cases() + assert_runall(runner) + assert 1 == len(runner.stats.failures(run=0)) + assert pass_run_no == rt.runtime().current_run + assert 0 == len(runner.stats.failures()) + + +def test_sigterm_handling(make_runner, make_cases, common_exec_ctx): + runner = make_runner() + with pytest.raises(ReframeForceExitError, + match='received TERM signal'): + runner.runall(make_cases([SelfKillCheck()])) + + assert_all_dead(runner) + assert runner.stats.num_cases() == 1 + assert len(runner.stats.failures()) == 1 + + +@pytest.fixture +def dep_checks(make_loader): + return make_loader( + ['unittests/resources/checks_unlisted/deps_complex.py'] + ).load_all() + + +@pytest.fixture +def dep_cases(dep_checks, make_cases): + return make_cases(dep_checks, sort=True) + + +def assert_dependency_run(runner): + assert_runall(runner) + stats = runner.stats + assert 10 == stats.num_cases(0) + assert 4 == len(stats.failures()) + for tf in stats.failures(): + check = tf.testcase.check + _, exc_value, _ = tf.exc_info + if check.name == 'T7' or check.name == 'T9': + assert isinstance(exc_value, TaskDependencyError) + + # Check that cleanup is executed properly for successful tests as well + for t in stats.tasks(): + check = t.testcase.check + if t.failed: + continue + + if t.ref_count == 0: + assert os.path.exists(os.path.join(check.outputdir, 'out.txt')) + + +def test_dependencies(make_runner, dep_cases, common_exec_ctx): + runner = make_runner() + runner.runall(dep_cases) + assert_dependency_run(runner) + + +def test_dependencies_with_retries(make_runner, dep_cases, common_exec_ctx): + runner = make_runner(max_retries=2) + runner.runall(dep_cases) + assert_dependency_run(runner) + + +class _TaskEventMonitor(executors.TaskEventListener): '''Event listener for monitoring the execution of the asynchronous execution policy. @@ -318,618 +351,239 @@ def on_task_setup(self, task): pass -class TestAsynchronousExecutionPolicy(TestSerialExecutionPolicy): - def setUp(self): - super().setUp() - self.runner = executors.Runner(policies.AsynchronousExecutionPolicy()) - self.runner.policy.keep_stage_files = True - self.monitor = TaskEventMonitor() - self.runner.policy.task_listeners.append(self.monitor) - - def set_max_jobs(self, value): - for p in rt.runtime().system.partitions: - p._max_jobs = value - - def read_timestamps(self, tasks): - '''Read the timestamps and sort them to permit simple - concurrency tests.''' - from reframe.utility.sanity import evaluate - - self.begin_stamps = [] - self.end_stamps = [] - for t in tasks: - with os_ext.change_dir(t.check.stagedir): - with open(evaluate(t.check.stdout), 'r') as f: - self.begin_stamps.append(float(f.readline().strip())) - self.end_stamps.append(float(f.readline().strip())) - - self.begin_stamps.sort() - self.end_stamps.sort() - - def test_concurrency_unlimited(self): - checks = [SleepCheck(0.5) for i in range(3)] - self.set_max_jobs(len(checks)) - self.runall(checks) - - # Ensure that all tests were run and without failures. - assert len(checks) == self.runner.stats.num_cases() - self.assertRunall() - assert 0 == len(self.runner.stats.failures()) - - # Ensure that maximum concurrency was reached as fast as possible - assert len(checks) == max(self.monitor.num_tasks) - assert len(checks) == self.monitor.num_tasks[len(checks)] - - self.read_timestamps(self.monitor.tasks) - - # Warn if not all tests were run in parallel; the corresponding strict - # check would be: - # - # self.assertTrue(self.begin_stamps[-1] <= self.end_stamps[0]) - # - if self.begin_stamps[-1] > self.end_stamps[0]: - pytest.skip('the system seems too much loaded.') - - def test_concurrency_limited(self): - # The number of checks must be <= 2*max_jobs. - checks = [SleepCheck(0.5) for i in range(5)] - max_jobs = len(checks) - 2 - self.set_max_jobs(max_jobs) - self.runall(checks) - - # Ensure that all tests were run and without failures. - assert len(checks) == self.runner.stats.num_cases() - self.assertRunall() - assert 0 == len(self.runner.stats.failures()) - - # Ensure that maximum concurrency was reached as fast as possible - assert max_jobs == max(self.monitor.num_tasks) - assert max_jobs == self.monitor.num_tasks[max_jobs] - - self.read_timestamps(self.monitor.tasks) - - # Ensure that the jobs after the first #max_jobs were each run after - # one of the previous #max_jobs jobs had finished - # (e.g. begin[max_jobs] > end[0]). - # Note: we may ensure this strictly as we may ensure serial behaviour. - begin_after_end = (b > e for b, e in zip(self.begin_stamps[max_jobs:], - self.end_stamps[:-max_jobs])) - assert all(begin_after_end) - - # NOTE: to ensure that these remaining jobs were also run - # in parallel one could do the command hereafter; however, it would - # require to substantially increase the sleep time (in SleepCheck), - # because of the delays in rescheduling (1s, 2s, 3s, 1s, 2s,...). - # We currently prefer not to do this last concurrency test to avoid an - # important prolongation of the unit test execution time. - # self.assertTrue(self.begin_stamps[-1] < self.end_stamps[max_jobs]) - - # Warn if the first #max_jobs jobs were not run in parallel; the - # corresponding strict check would be: - # self.assertTrue(self.begin_stamps[max_jobs-1] <= self.end_stamps[0]) - if self.begin_stamps[max_jobs-1] > self.end_stamps[0]: - pytest.skip('the system seems too loaded.') - - def test_concurrency_none(self): - checks = [SleepCheck(0.5) for i in range(3)] - num_checks = len(checks) - self.set_max_jobs(1) - self.runall(checks) - - # Ensure that all tests were run and without failures. - assert len(checks) == self.runner.stats.num_cases() - self.assertRunall() - assert 0 == len(self.runner.stats.failures()) - - # Ensure that a single task was running all the time - assert 1 == max(self.monitor.num_tasks) - - # Read the timestamps sorted to permit simple concurrency tests. - self.read_timestamps(self.monitor.tasks) - - # Ensure that the jobs were run after the previous job had finished - # (e.g. begin[1] > end[0]). - begin_after_end = (b > e for b, e in zip(self.begin_stamps[1:], - self.end_stamps[:-1])) - assert all(begin_after_end) - - def _run_checks(self, checks, max_jobs): - self.set_max_jobs(max_jobs) - with pytest.raises(KeyboardInterrupt): - self.runall(checks) - - assert 4 == self.runner.stats.num_cases() - self.assertRunall() - assert 4 == len(self.runner.stats.failures()) - self.assert_all_dead() - - def test_kbd_interrupt_in_wait_with_concurrency(self): - checks = [KeyboardInterruptCheck(), - SleepCheck(10), SleepCheck(10), SleepCheck(10)] - self._run_checks(checks, 4) - - def test_kbd_interrupt_in_wait_with_limited_concurrency(self): - # The general idea for this test is to allow enough time for all the - # four checks to be submitted and at the same time we need the - # KeyboardInterruptCheck to finish first (the corresponding wait should - # trigger the failure), so as to make the framework kill the remaining - # three. - checks = [KeyboardInterruptCheck(), - SleepCheck(10), SleepCheck(10), SleepCheck(10)] - self._run_checks(checks, 2) - - def test_kbd_interrupt_in_setup_with_concurrency(self): - checks = [SleepCheck(1), SleepCheck(1), SleepCheck(1), - KeyboardInterruptCheck(phase='setup')] - self._run_checks(checks, 4) - - def test_kbd_interrupt_in_setup_with_limited_concurrency(self): - checks = [SleepCheck(1), SleepCheck(1), SleepCheck(1), - KeyboardInterruptCheck(phase='setup')] - self._run_checks(checks, 2) - - def test_poll_fails_main_loop(self): - num_tasks = 3 - checks = [SleepCheckPollFail(10) for i in range(num_tasks)] - num_checks = len(checks) - self.set_max_jobs(1) - self.runall(checks) - stats = self.runner.stats - assert num_tasks == stats.num_cases() - self.assertRunall() - assert num_tasks == len(stats.failures()) - - def test_poll_fails_busy_loop(self): - num_tasks = 3 - checks = [SleepCheckPollFailLate(1/i) for i in range(1, num_tasks+1)] - num_checks = len(checks) - self.set_max_jobs(1) - self.runall(checks) - stats = self.runner.stats - assert num_tasks == stats.num_cases() - self.assertRunall() - assert num_tasks == len(stats.failures()) - - -class TestDependencies(unittest.TestCase): - class Node: - '''A node in the test case graph. - - It's simply a wrapper to a (test_name, partition, environment) tuple - that can interact seemlessly with a real test case. - It's meant for convenience in unit testing. - ''' - - def __init__(self, cname, pname, ename): - self.cname, self.pname, self.ename = cname, pname, ename - - def __eq__(self, other): - if isinstance(other, type(self)): - return (self.cname == other.cname and - self.pname == other.pname and - self.ename == other.ename) - - if isinstance(other, executors.TestCase): - return (self.cname == other.check.name and - self.pname == other.partition.fullname and - self.ename == other.environ.name) - - return NotImplemented - - def __hash__(self): - return hash(self.cname) ^ hash(self.pname) ^ hash(self.ename) - - def __repr__(self): - return 'Node(%r, %r, %r)' % (self.cname, self.pname, self.ename) - - def has_edge(graph, src, dst): - return dst in graph[src] - - def num_deps(graph, cname): - return sum(len(deps) for c, deps in graph.items() - if c.check.name == cname) - - def in_degree(graph, node): - for v in graph.keys(): - if v == node: - return v.num_dependents - - def find_check(name, checks): - for c in checks: - if c.name == name: - return c - - return None - - def find_case(cname, ename, cases): - for c in cases: - if c.check.name == cname and c.environ.name == ename: - return c - - def setUp(self): - self.loader = RegressionCheckLoader([ - 'unittests/resources/checks_unlisted/deps_simple.py' - ]) - - # Set runtime prefix - rt.runtime().resources.prefix = tempfile.mkdtemp(dir='unittests') - - def tearDown(self): - os_ext.rmtree(rt.runtime().resources.prefix) - - @rt.switch_runtime(fixtures.TEST_SITE_CONFIG, 'sys0') - def test_eq_hash(self): - find_case = TestDependencies.find_case - cases = executors.generate_testcases(self.loader.load_all()) - - case0 = find_case('Test0', 'e0', cases) - case1 = find_case('Test0', 'e1', cases) - case0_copy = case0.clone() - - assert case0 == case0_copy - assert hash(case0) == hash(case0_copy) - assert case1 != case0 - assert hash(case1) != hash(case0) - - @rt.switch_runtime(fixtures.TEST_SITE_CONFIG, 'sys0') - def test_build_deps(self): - Node = TestDependencies.Node - has_edge = TestDependencies.has_edge - num_deps = TestDependencies.num_deps - in_degree = TestDependencies.in_degree - find_check = TestDependencies.find_check - find_case = TestDependencies.find_case - - checks = self.loader.load_all() - cases = executors.generate_testcases(checks) - - # Test calling getdep() before having built the graph - t = find_check('Test1_exact', checks) - with pytest.raises(DependencyError): - t.getdep('Test0', 'e0') - - # Build dependencies and continue testing - deps = dependency.build_deps(cases) - dependency.validate_deps(deps) - - # Check DEPEND_FULLY dependencies - assert num_deps(deps, 'Test1_fully') == 8 - for p in ['sys0:p0', 'sys0:p1']: - for e0 in ['e0', 'e1']: - for e1 in ['e0', 'e1']: - assert has_edge(deps, - Node('Test1_fully', p, e0), - Node('Test0', p, e1)) - - # Check DEPEND_BY_ENV - assert num_deps(deps, 'Test1_by_env') == 4 - assert num_deps(deps, 'Test1_default') == 4 - for p in ['sys0:p0', 'sys0:p1']: - for e in ['e0', 'e1']: - assert has_edge(deps, - Node('Test1_by_env', p, e), - Node('Test0', p, e)) - assert has_edge(deps, - Node('Test1_default', p, e), - Node('Test0', p, e)) - - # Check DEPEND_EXACT - assert num_deps(deps, 'Test1_exact') == 6 - for p in ['sys0:p0', 'sys0:p1']: - assert has_edge(deps, - Node('Test1_exact', p, 'e0'), - Node('Test0', p, 'e0')) - assert has_edge(deps, - Node('Test1_exact', p, 'e0'), - Node('Test0', p, 'e1')) - assert has_edge(deps, - Node('Test1_exact', p, 'e1'), - Node('Test0', p, 'e1')) - - # Check in-degree of Test0 - - # 2 from Test1_fully, - # 1 from Test1_by_env, - # 1 from Test1_exact, - # 1 from Test1_default - assert in_degree(deps, Node('Test0', 'sys0:p0', 'e0')) == 5 - assert in_degree(deps, Node('Test0', 'sys0:p1', 'e0')) == 5 - - # 2 from Test1_fully, - # 1 from Test1_by_env, - # 2 from Test1_exact, - # 1 from Test1_default - assert in_degree(deps, Node('Test0', 'sys0:p0', 'e1')) == 6 - assert in_degree(deps, Node('Test0', 'sys0:p1', 'e1')) == 6 - - # Pick a check to test getdep() - check_e0 = find_case('Test1_exact', 'e0', cases).check - check_e1 = find_case('Test1_exact', 'e1', cases).check - - with pytest.raises(DependencyError): - check_e0.getdep('Test0') - - # Set the current environment - check_e0._current_environ = Environment('e0') - check_e1._current_environ = Environment('e1') - - assert check_e0.getdep('Test0', 'e0').name == 'Test0' - assert check_e0.getdep('Test0', 'e1').name == 'Test0' - assert check_e1.getdep('Test0', 'e1').name == 'Test0' - with pytest.raises(DependencyError): - check_e0.getdep('TestX', 'e0') - - with pytest.raises(DependencyError): - check_e0.getdep('Test0', 'eX') - - with pytest.raises(DependencyError): - check_e1.getdep('Test0', 'e0') - - @rt.switch_runtime(fixtures.TEST_SITE_CONFIG, 'sys0') - def test_build_deps_unknown_test(self): - find_check = TestDependencies.find_check - checks = self.loader.load_all() - - # Add some inexistent dependencies - test0 = find_check('Test0', checks) - for depkind in ('default', 'fully', 'by_env', 'exact'): - test1 = find_check('Test1_' + depkind, checks) - if depkind == 'default': - test1.depends_on('TestX') - elif depkind == 'exact': - test1.depends_on('TestX', rfm.DEPEND_EXACT, {'e0': ['e0']}) - elif depkind == 'fully': - test1.depends_on('TestX', rfm.DEPEND_FULLY) - elif depkind == 'by_env': - test1.depends_on('TestX', rfm.DEPEND_BY_ENV) - - with pytest.raises(DependencyError): - dependency.build_deps(executors.generate_testcases(checks)) - - @rt.switch_runtime(fixtures.TEST_SITE_CONFIG, 'sys0') - def test_build_deps_unknown_target_env(self): - find_check = TestDependencies.find_check - checks = self.loader.load_all() - - # Add some inexistent dependencies - test0 = find_check('Test0', checks) - test1 = find_check('Test1_default', checks) - test1.depends_on('Test0', rfm.DEPEND_EXACT, {'e0': ['eX']}) - with pytest.raises(DependencyError): - dependency.build_deps(executors.generate_testcases(checks)) - - @rt.switch_runtime(fixtures.TEST_SITE_CONFIG, 'sys0') - def test_build_deps_unknown_source_env(self): - find_check = TestDependencies.find_check - num_deps = TestDependencies.num_deps - checks = self.loader.load_all() - - # Add some inexistent dependencies - test0 = find_check('Test0', checks) - test1 = find_check('Test1_default', checks) - test1.depends_on('Test0', rfm.DEPEND_EXACT, {'eX': ['e0']}) - - # Unknown source is ignored, because it might simply be that the test - # is not executed for eX - deps = dependency.build_deps(executors.generate_testcases(checks)) - assert num_deps(deps, 'Test1_default') == 4 - - @rt.switch_runtime(fixtures.TEST_SITE_CONFIG, 'sys0') - def test_build_deps_empty(self): - assert {} == dependency.build_deps([]) - - def create_test(self, name): - test = rfm.RegressionTest() - test.name = name - test.valid_systems = ['*'] - test.valid_prog_environs = ['*'] - test.executable = 'echo' - test.executable_opts = [name] - return test - - @rt.switch_runtime(fixtures.TEST_SITE_CONFIG, 'sys0') - def test_valid_deps(self): - # - # t0 +-->t5<--+ - # ^ | | - # | | | - # +-->t1<--+ t6 t7 - # | | ^ - # t2<------t3 | - # ^ ^ | - # | | t8 - # +---t4---+ - # - t0 = self.create_test('t0') - t1 = self.create_test('t1') - t2 = self.create_test('t2') - t3 = self.create_test('t3') - t4 = self.create_test('t4') - t5 = self.create_test('t5') - t6 = self.create_test('t6') - t7 = self.create_test('t7') - t8 = self.create_test('t8') - t1.depends_on('t0') - t2.depends_on('t1') - t3.depends_on('t1') - t3.depends_on('t2') - t4.depends_on('t2') - t4.depends_on('t3') - t6.depends_on('t5') - t7.depends_on('t5') - t8.depends_on('t7') - dependency.validate_deps( - dependency.build_deps( - executors.generate_testcases([t0, t1, t2, t3, t4, - t5, t6, t7, t8]) - ) - ) - - @rt.switch_runtime(fixtures.TEST_SITE_CONFIG, 'sys0') - def test_cyclic_deps(self): - # - # t0 +-->t5<--+ - # ^ | | - # | | | - # +-->t1<--+ t6 t7 - # | | | ^ - # t2 | t3 | - # ^ | ^ | - # | v | t8 - # +---t4---+ - # - t0 = self.create_test('t0') - t1 = self.create_test('t1') - t2 = self.create_test('t2') - t3 = self.create_test('t3') - t4 = self.create_test('t4') - t5 = self.create_test('t5') - t6 = self.create_test('t6') - t7 = self.create_test('t7') - t8 = self.create_test('t8') - t1.depends_on('t0') - t1.depends_on('t4') - t2.depends_on('t1') - t3.depends_on('t1') - t4.depends_on('t2') - t4.depends_on('t3') - t6.depends_on('t5') - t7.depends_on('t5') - t8.depends_on('t7') - deps = dependency.build_deps( - executors.generate_testcases([t0, t1, t2, t3, t4, - t5, t6, t7, t8]) - ) - - with pytest.raises(DependencyError) as exc_info: - dependency.validate_deps(deps) - - assert ('t4->t2->t1->t4' in str(exc_info.value) or - 't2->t1->t4->t2' in str(exc_info.value) or - 't1->t4->t2->t1' in str(exc_info.value) or - 't1->t4->t3->t1' in str(exc_info.value) or - 't4->t3->t1->t4' in str(exc_info.value) or - 't3->t1->t4->t3' in str(exc_info.value)) - - @rt.switch_runtime(fixtures.TEST_SITE_CONFIG, 'sys0') - def test_cyclic_deps_by_env(self): - t0 = self.create_test('t0') - t1 = self.create_test('t1') - t1.depends_on('t0', rfm.DEPEND_EXACT, {'e0': ['e0']}) - t0.depends_on('t1', rfm.DEPEND_EXACT, {'e1': ['e1']}) - deps = dependency.build_deps( - executors.generate_testcases([t0, t1]) - ) - with pytest.raises(DependencyError) as exc_info: - dependency.validate_deps(deps) - - assert ('t1->t0->t1' in str(exc_info.value) or - 't0->t1->t0' in str(exc_info.value)) - - @rt.switch_runtime(fixtures.TEST_SITE_CONFIG, 'sys0') - def test_validate_deps_empty(self): - dependency.validate_deps({}) - - def assert_topological_order(self, cases, graph): - cases_order = [] - visited_tests = set() - tests = util.OrderedSet() - for c in cases: - check, part, env = c - cases_order.append((check.name, part.fullname, env.name)) - tests.add(check.name) - visited_tests.add(check.name) - - # Assert that all dependencies of c have been visited before - for d in graph[c]: - if d not in cases: - # dependency points outside the subgraph - continue - - assert d.check.name in visited_tests - - # Check the order of systems and prog. environments - # We are checking against all possible orderings - valid_orderings = [] - for partitions in itertools.permutations(['sys0:p0', 'sys0:p1']): - for environs in itertools.permutations(['e0', 'e1']): - ordering = [] - for t in tests: - for p in partitions: - for e in environs: - ordering.append((t, p, e)) - - valid_orderings.append(ordering) - - assert cases_order in valid_orderings - - @rt.switch_runtime(fixtures.TEST_SITE_CONFIG, 'sys0') - def test_toposort(self): - # - # t0 +-->t5<--+ - # ^ | | - # | | | - # +-->t1<--+ t6 t7 - # | | ^ - # t2<------t3 | - # ^ ^ | - # | | t8 - # +---t4---+ - # - t0 = self.create_test('t0') - t1 = self.create_test('t1') - t2 = self.create_test('t2') - t3 = self.create_test('t3') - t4 = self.create_test('t4') - t5 = self.create_test('t5') - t6 = self.create_test('t6') - t7 = self.create_test('t7') - t8 = self.create_test('t8') - t1.depends_on('t0') - t2.depends_on('t1') - t3.depends_on('t1') - t3.depends_on('t2') - t4.depends_on('t2') - t4.depends_on('t3') - t6.depends_on('t5') - t7.depends_on('t5') - t8.depends_on('t7') - deps = dependency.build_deps( - executors.generate_testcases([t0, t1, t2, t3, t4, - t5, t6, t7, t8]) - ) - cases = dependency.toposort(deps) - self.assert_topological_order(cases, deps) - - @rt.switch_runtime(fixtures.TEST_SITE_CONFIG, 'sys0') - def test_toposort_subgraph(self): - # - # t0 - # ^ - # | - # +-->t1<--+ - # | | - # t2<------t3 - # ^ ^ - # | | - # +---t4---+ - # - t0 = self.create_test('t0') - t1 = self.create_test('t1') - t2 = self.create_test('t2') - t3 = self.create_test('t3') - t4 = self.create_test('t4') - t1.depends_on('t0') - t2.depends_on('t1') - t3.depends_on('t1') - t3.depends_on('t2') - t4.depends_on('t2') - t4.depends_on('t3') - full_deps = dependency.build_deps( - executors.generate_testcases([t0, t1, t2, t3, t4]) - ) - partial_deps = dependency.build_deps( - executors.generate_testcases([t3, t4]), full_deps - ) - cases = dependency.toposort(partial_deps, is_subgraph=True) - self.assert_topological_order(cases, partial_deps) +@pytest.fixture +def make_async_exec_ctx(temp_runtime): + def _make_async_exec_ctx(max_jobs): + yield from temp_runtime(fixtures.TEST_CONFIG_FILE, 'generic', + {'systems/partitions/max_jobs': max_jobs}) + + return _make_async_exec_ctx + + +@pytest.fixture +def async_runner(): + evt_monitor = _TaskEventMonitor() + ret = executors.Runner(policies.AsynchronousExecutionPolicy()) + ret.policy.keep_stage_files = True + ret.policy.task_listeners.append(evt_monitor) + return ret, evt_monitor + + +def _read_timestamps(tasks): + '''Read the timestamps and sort them to permit simple + concurrency tests.''' + from reframe.utility.sanity import evaluate + + begin_stamps = [] + end_stamps = [] + for t in tasks: + with os_ext.change_dir(t.check.stagedir): + with open(evaluate(t.check.stdout), 'r') as f: + begin_stamps.append(float(f.readline().strip())) + end_stamps.append(float(f.readline().strip())) + + begin_stamps.sort() + end_stamps.sort() + return begin_stamps, end_stamps + + +def test_concurrency_unlimited(async_runner, make_cases, make_async_exec_ctx): + num_checks = 3 + + # Trigger evaluation of the execution context + ctx = make_async_exec_ctx(num_checks) + next(ctx) + + runner, monitor = async_runner + runner.runall(make_cases([SleepCheck(.5) for i in range(num_checks)])) + + # Ensure that all tests were run and without failures. + assert num_checks == runner.stats.num_cases() + assert_runall(runner) + assert 0 == len(runner.stats.failures()) + + # Ensure that maximum concurrency was reached as fast as possible + assert num_checks == max(monitor.num_tasks) + assert num_checks == monitor.num_tasks[num_checks] + begin_stamps, end_stamps = _read_timestamps(monitor.tasks) + + # Warn if not all tests were run in parallel; the corresponding strict + # check would be: + # + # assert begin_stamps[-1] <= end_stamps[0] + # + if begin_stamps[-1] > end_stamps[0]: + pytest.skip('the system seems too much loaded.') + + +def test_concurrency_limited(async_runner, make_cases, make_async_exec_ctx): + # The number of checks must be <= 2*max_jobs. + num_checks, max_jobs = 5, 3 + ctx = make_async_exec_ctx(max_jobs) + next(ctx) + + runner, monitor = async_runner + runner.runall(make_cases([SleepCheck(.5) for i in range(num_checks)])) + + # Ensure that all tests were run and without failures. + assert num_checks == runner.stats.num_cases() + assert_runall(runner) + assert 0 == len(runner.stats.failures()) + + # Ensure that maximum concurrency was reached as fast as possible + assert max_jobs == max(monitor.num_tasks) + assert max_jobs == monitor.num_tasks[max_jobs] + + begin_stamps, end_stamps = _read_timestamps(monitor.tasks) + + # Ensure that the jobs after the first #max_jobs were each run after + # one of the previous #max_jobs jobs had finished + # (e.g. begin[max_jobs] > end[0]). + # Note: we may ensure this strictly as we may ensure serial behaviour. + begin_after_end = (b > e for b, e in zip(begin_stamps[max_jobs:], + end_stamps[:-max_jobs])) + assert all(begin_after_end) + + # NOTE: to ensure that these remaining jobs were also run + # in parallel one could do the command hereafter; however, it would + # require to substantially increase the sleep time (in SleepCheck), + # because of the delays in rescheduling (1s, 2s, 3s, 1s, 2s,...). + # We currently prefer not to do this last concurrency test to avoid an + # important prolongation of the unit test execution time. + # self.assertTrue(self.begin_stamps[-1] < self.end_stamps[max_jobs]) + + # Warn if the first #max_jobs jobs were not run in parallel; the + # corresponding strict check would be: + # self.assertTrue(self.begin_stamps[max_jobs-1] <= self.end_stamps[0]) + if begin_stamps[max_jobs-1] > end_stamps[0]: + pytest.skip('the system seems too loaded.') + + +def test_concurrency_none(async_runner, make_cases, make_async_exec_ctx): + num_checks = 3 + ctx = make_async_exec_ctx(1) + next(ctx) + + runner, monitor = async_runner + runner.runall(make_cases([SleepCheck(.5) for i in range(num_checks)])) + + # Ensure that all tests were run and without failures. + assert num_checks == runner.stats.num_cases() + assert_runall(runner) + assert 0 == len(runner.stats.failures()) + + # Ensure that a single task was running all the time + assert 1 == max(monitor.num_tasks) + + # Read the timestamps sorted to permit simple concurrency tests. + begin_stamps, end_stamps = _read_timestamps(monitor.tasks) + + # Ensure that the jobs were run after the previous job had finished + # (e.g. begin[1] > end[0]). + begin_after_end = (b > e + for b, e in zip(begin_stamps[1:], end_stamps[:-1])) + assert all(begin_after_end) + + +def assert_interrupted_run(runner): + assert 4 == runner.stats.num_cases() + assert_runall(runner) + assert 4 == len(runner.stats.failures()) + assert_all_dead(runner) + + +def test_kbd_interrupt_in_wait_with_concurrency(async_runner, make_cases, + make_async_exec_ctx): + ctx = make_async_exec_ctx(4) + next(ctx) + + runner, _ = async_runner + with pytest.raises(KeyboardInterrupt): + runner.runall(make_cases([ + KeyboardInterruptCheck(), SleepCheck(10), + SleepCheck(10), SleepCheck(10) + ])) + + assert_interrupted_run(runner) + + +def test_kbd_interrupt_in_wait_with_limited_concurrency( + async_runner, make_cases, make_async_exec_ctx): + # The general idea for this test is to allow enough time for all the + # four checks to be submitted and at the same time we need the + # KeyboardInterruptCheck to finish first (the corresponding wait should + # trigger the failure), so as to make the framework kill the remaining + # three. + ctx = make_async_exec_ctx(2) + next(ctx) + + runner, _ = async_runner + with pytest.raises(KeyboardInterrupt): + runner.runall(make_cases([ + KeyboardInterruptCheck(), SleepCheck(10), + SleepCheck(10), SleepCheck(10) + ])) + + assert_interrupted_run(runner) + + +def test_kbd_interrupt_in_setup_with_concurrency(async_runner, make_cases, + make_async_exec_ctx): + ctx = make_async_exec_ctx(4) + next(ctx) + + runner, _ = async_runner + with pytest.raises(KeyboardInterrupt): + runner.runall(make_cases([ + SleepCheck(1), SleepCheck(1), SleepCheck(1), + KeyboardInterruptCheck(phase='setup') + ])) + + assert_interrupted_run(runner) + + +def test_kbd_interrupt_in_setup_with_limited_concurrency( + async_runner, make_cases, make_async_exec_ctx): + ctx = make_async_exec_ctx(2) + next(ctx) + + runner, _ = async_runner + with pytest.raises(KeyboardInterrupt): + runner.runall(make_cases([ + SleepCheck(1), SleepCheck(1), SleepCheck(1), + KeyboardInterruptCheck(phase='setup') + ])) + + assert_interrupted_run(runner) + + +def test_poll_fails_in_main_loop(async_runner, make_cases, + make_async_exec_ctx): + ctx = make_async_exec_ctx(1) + next(ctx) + + runner, _ = async_runner + num_checks = 3 + runner.runall(make_cases([SleepCheckPollFail(10) + for i in range(num_checks)])) + + stats = runner.stats + assert num_checks == stats.num_cases() + assert_runall(runner) + assert num_checks == len(stats.failures()) + + +def test_poll_fails_in_busy_loop(async_runner, make_cases, + make_async_exec_ctx): + ctx = make_async_exec_ctx(1) + next(ctx) + + runner, _ = async_runner + num_checks = 3 + runner.runall(make_cases([SleepCheckPollFailLate(1/i) + for i in range(1, num_checks+1)])) + + stats = runner.stats + assert num_checks == stats.num_cases() + assert_runall(runner) + assert num_checks == len(stats.failures()) diff --git a/unittests/test_runtime.py b/unittests/test_runtime.py deleted file mode 100644 index 961efa3bc2..0000000000 --- a/unittests/test_runtime.py +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright 2016-2020 Swiss National Supercomputing Centre (CSCS/ETH Zurich) -# ReFrame Project Developers. See the top-level LICENSE file for details. -# -# SPDX-License-Identifier: BSD-3-Clause - -import unittest - -import reframe.core.runtime as rt -import unittests.fixtures as fixtures - - -class TestRuntime(unittest.TestCase): - @rt.switch_runtime(fixtures.TEST_SITE_CONFIG, 'testsys') - def test_hostsystem_api(self): - system = rt.runtime().system - assert 'testsys' == system.name - assert 'Fake system for unit tests' == system.descr - assert 2 == len(system.partitions) - assert system.partition('login') is not None - assert system.partition('gpu') is not None - assert system.partition('foobar') is None - - # Test delegation to the underlying System - assert '.rfm_testing' == system.prefix - assert '.rfm_testing/resources' == system.resourcesdir - assert '.rfm_testing/perflogs' == system.perflogdir diff --git a/unittests/test_schedulers.py b/unittests/test_schedulers.py index 8442277a01..5a8599f4ac 100644 --- a/unittests/test_schedulers.py +++ b/unittests/test_schedulers.py @@ -4,6 +4,7 @@ # SPDX-License-Identifier: BSD-3-Clause import abc +import functools import os import pytest import re @@ -16,896 +17,508 @@ import reframe.core.runtime as rt import reframe.utility.os_ext as os_ext import unittests.fixtures as fixtures +from reframe.core.backends import (getlauncher, getscheduler) from reframe.core.environments import Environment from reframe.core.exceptions import JobError, JobNotStartedError from reframe.core.launchers.local import LocalLauncher -from reframe.core.launchers.registry import getlauncher from reframe.core.schedulers import Job -from reframe.core.schedulers.registry import getscheduler from reframe.core.schedulers.slurm import _SlurmNode, _create_nodes -class _TestJob(abc.ABC): - def setUp(self): - self.workdir = tempfile.mkdtemp(dir='unittests') - self.testjob = Job.create( - self.scheduler, self.launcher, +@pytest.fixture +def launcher(): + return getlauncher('local') + + +@pytest.fixture(params=['slurm', 'squeue', 'local', 'pbs', 'torque']) +def scheduler(request): + return getscheduler(request.param) + + +@pytest.fixture +def slurm_only(scheduler): + if scheduler.registered_name not in ('slurm', 'squeue'): + pytest.skip(f'test is relevant only for Slurm backends') + + +@pytest.fixture +def local_only(scheduler): + if scheduler.registered_name != 'local': + pytest.skip(f'test is relevant only for the local scheduler') + + +@pytest.fixture +def temp_runtime(tmp_path): + def _temp_runtime(site_config, system=None, options={}): + options.update({'systems/prefix': tmp_path}) + with rt.temp_runtime(site_config, system, options): + yield rt.runtime + + yield _temp_runtime + + +@pytest.fixture +def exec_ctx(temp_runtime, scheduler): + if fixtures.USER_CONFIG_FILE and scheduler.registered_name != 'local': + rt = temp_runtime(fixtures.USER_CONFIG_FILE, fixtures.USER_SYSTEM) + else: + rt = temp_runtime(fixtures.TEST_CONFIG_FILE, 'generic') + + next(rt) + if scheduler.registered_name == 'squeue': + # slurm backend fulfills the functionality of the squeue backend, so + # if squeue is not configured, use slurrm instead + partition = (fixtures.partition_by_scheduler('squeue') or + fixtures.partition_by_scheduler('slurm')) + else: + partition = fixtures.partition_by_scheduler(scheduler.registered_name) + + if partition is None: + pytest.skip( + f"scheduler '{scheduler.registered_name}' not configured" + ) + + return partition + + +@pytest.fixture +def make_job(scheduler, launcher, tmp_path): + def _make_job(**jobargs): + return Job.create( + scheduler(), launcher(), name='testjob', - workdir=self.workdir, - script_filename=os_ext.mkstemp_path( - dir=self.workdir, suffix='.sh' - ), - stdout=os_ext.mkstemp_path(dir=self.workdir, suffix='.out'), - stderr=os_ext.mkstemp_path(dir=self.workdir, suffix='.err'), + workdir=tmp_path, + script_filename=str(tmp_path / 'job.sh'), + stdout=str(tmp_path / 'job.out'), + stderr=str(tmp_path / 'job.err'), + **jobargs + ) + + return _make_job + + +@pytest.fixture +def minimal_job(make_job): + return make_job() + + +@pytest.fixture +def fake_job(make_job): + ret = make_job( + sched_nodelist='nid000[00-17]', + sched_exclude_nodelist='nid00016', + sched_partition='foo', + sched_reservation='bar', + sched_account='spam', + sched_exclusive_access=True + ) + ret.time_limit = '5m' + ret.num_tasks = 16 + ret.num_tasks_per_node = 2 + ret.num_tasks_per_core = 1 + ret.num_tasks_per_socket = 1 + ret.num_cpus_per_task = 18 + ret.use_smt = True + ret.options = ['--gres=gpu:4', + '#DW jobdw capacity=100GB', + '#DW stage_in source=/foo'] + return ret + + +def prepare_job(job, command='hostname', pre_run=None, post_run=None): + environs = [Environment(name='foo', modules=['testmod_foo'])] + pre_run = pre_run or ['echo prerun'] + post_run = post_run or ['echo postrun'] + with rt.module_use('unittests/modules'): + job.prepare( + [ + *pre_run, + job.launcher.run_command(job) + ' ' + command, + post_run + ], + environs ) - self.environs = [Environment(name='foo', modules=['testmod_foo'])] - self.pre_run = ['echo prerun'] - self.post_run = ['echo postrun'] - self.parallel_cmd = 'hostname' - - def tearDown(self): - os_ext.rmtree(self.workdir) - - def prepare(self): - with rt.module_use('unittests/modules'): - self.testjob.prepare(self.commands, self.environs) - - @property - def commands(self): - runcmd = self.launcher.run_command(self.testjob) - return [*self.pre_run, - runcmd + ' ' + self.parallel_cmd, - *self.post_run] - - @property - def scheduler(self): - return getscheduler(self.sched_name)() - - @property - @abc.abstractmethod - def sched_name(self): - '''Return the registered name of the scheduler.''' - - @property - def sched_configured(self): - return True - - @property - def launcher(self): - return getlauncher(self.launcher_name)() - - @property - @abc.abstractmethod - def launcher_name(self): - '''Return the registered name of the launcher.''' - - @abc.abstractmethod - def setup_user(self, msg=None): - '''Configure the test for running with the user supplied job scheduler - configuration or skip it. - ''' - partition = fixtures.partition_with_scheduler(self.sched_name) - if partition is None: - msg = msg or "scheduler '%s' not configured" % self.sched_name - pytest.skip(msg) - - self.testjob._sched_access = partition.access - - def assertScriptSanity(self, script_file): - '''Assert the sanity of the produced script file.''' - with open(self.testjob.script_filename) as fp: - matches = re.findall(r'echo prerun|echo postrun|hostname', - fp.read()) - assert ['echo prerun', 'hostname', 'echo postrun'] == matches - - def setup_job(self): - # Mock up a job submission - self.testjob.time_limit = '5m' - self.testjob.num_tasks = 16 - self.testjob.num_tasks_per_node = 2 - self.testjob.num_tasks_per_core = 1 - self.testjob.num_tasks_per_socket = 1 - self.testjob.num_cpus_per_task = 18 - self.testjob.use_smt = True - self.testjob.options = ['--gres=gpu:4', - '#DW jobdw capacity=100GB', - '#DW stage_in source=/foo'] - self.testjob._sched_nodelist = 'nid000[00-17]' - self.testjob._sched_exclude_nodelist = 'nid00016' - self.testjob._sched_partition = 'foo' - self.testjob._sched_reservation = 'bar' - self.testjob._sched_account = 'spam' - self.testjob._sched_exclusive_access = True - - def test_prepare(self): - self.prepare() - self.assertScriptSanity(self.testjob.script_filename) - - @fixtures.switch_to_user_runtime - def test_submit(self): - self.setup_user() - self.prepare() - assert self.testjob.nodelist is None - self.testjob.submit() - assert self.testjob.jobid is not None - self.testjob.wait() - - @fixtures.switch_to_user_runtime - def test_submit_timelimit(self, check_elapsed_time=True): - self.setup_user() - self.parallel_cmd = 'sleep 10' - self.testjob.time_limit = '2s' - self.prepare() - t_job = datetime.now() - self.testjob.submit() - assert self.testjob.jobid is not None - self.testjob.wait() - t_job = datetime.now() - t_job - if check_elapsed_time: - assert t_job.total_seconds() >= 2 - assert t_job.total_seconds() < 3 - - with open(self.testjob.stdout) as fp: - assert re.search('postrun', fp.read()) is None - - @fixtures.switch_to_user_runtime - def test_cancel(self): - self.setup_user() - self.parallel_cmd = 'sleep 30' - self.prepare() - t_job = datetime.now() - self.testjob.submit() - self.testjob.cancel() - self.testjob.wait() - t_job = datetime.now() - t_job - assert self.testjob.finished() - assert t_job.total_seconds() < 30 - - def test_cancel_before_submit(self): - self.parallel_cmd = 'sleep 3' - self.prepare() - with pytest.raises(JobNotStartedError): - self.testjob.cancel() - - def test_wait_before_submit(self): - self.parallel_cmd = 'sleep 3' - self.prepare() - with pytest.raises(JobNotStartedError): - self.testjob.wait() - - @fixtures.switch_to_user_runtime - def test_poll(self): - self.setup_user() - self.parallel_cmd = 'sleep 2' - self.prepare() - self.testjob.submit() - assert not self.testjob.finished() - self.testjob.wait() - - def test_poll_before_submit(self): - self.parallel_cmd = 'sleep 3' - self.prepare() - with pytest.raises(JobNotStartedError): - self.testjob.finished() - - def test_no_empty_lines_in_preamble(self): - for l in self.testjob.scheduler.emit_preamble(self.testjob): - assert l != '' - - def test_guess_num_tasks(self): - self.testjob.num_tasks = 0 - with pytest.raises(NotImplementedError): - self.testjob.guess_num_tasks() - # Monkey patch `self._update_state` to simulate that the job is - # pending on the queue for enough time so it can be canceled due - # to exceeding the maximum pending time - @fixtures.switch_to_user_runtime - def test_submit_max_pending_time(self): - self.setup_user() - self.parallel_cmd = 'sleep 30' - self.prepare() - self.testjob.scheduler._update_state = self._update_state - self.testjob._max_pending_time = timedelta(milliseconds=50) - self.testjob.submit() - with pytest.raises(JobError, - match='maximum pending time exceeded'): - self.testjob.wait() - - -class TestLocalJob(_TestJob, unittest.TestCase): - def assertProcessDied(self, pid): - try: - os.kill(pid, 0) - pytest.fail('process %s is still alive' % pid) - except (ProcessLookupError, PermissionError): - pass - - @property - def sched_name(self): - return 'local' - - @property - def launcher_name(self): - return 'local' - - @property - def sched_configured(self): - return True - - def setup_user(self, msg=None): - # Local scheduler is by definition available - pass - def test_submit(self): - super().test_submit() - assert 0 == self.testjob.exitcode - assert [socket.gethostname()] == self.testjob.nodelist - - def test_submit_timelimit(self): - super().test_submit_timelimit() - assert self.testjob.state == 'TIMEOUT' - - def test_cancel_with_grace(self): - # This test emulates a spawned process that ignores the SIGTERM signal - # and also spawns another process: - # - # reframe --- local job script --- sleep 10 - # (TERM IGN) - # - # We expect the job not to be cancelled immediately, since it ignores - # the gracious signal we are sending it. However, we expect it to be - # killed immediately after the grace period of 2 seconds expires. - # - # We also check that the additional spawned process is also killed. - self.parallel_cmd = 'sleep 5 &' - self.pre_run = ['trap -- "" TERM'] - self.post_run = ['echo $!', 'wait'] - self.testjob.time_limit = '1m' - self.testjob.scheduler._cancel_grace_period = 2 - - self.prepare() - self.testjob.submit() - - # Stall a bit here to let the the spawned process start and install its - # signal handler for SIGTERM - time.sleep(1) - - t_grace = datetime.now() - self.testjob.cancel() - t_grace = datetime.now() - t_grace - - self.testjob.wait() - # Read pid of spawned sleep - with open(self.testjob.stdout) as f: - sleep_pid = int(f.read()) - - assert t_grace.total_seconds() >= 2 - assert t_grace.total_seconds() < 5 - assert self.testjob.state == 'TIMEOUT' - - # Verify that the spawned sleep is killed, too - self.assertProcessDied(sleep_pid) - - def test_cancel_term_ignore(self): - # This test emulates a descendant process of the spawned job that - # ignores the SIGTERM signal: - # - # reframe --- local job script --- sleep_deeply.sh --- sleep - # (TERM IGN) - # - # Since the "local job script" does not ignore SIGTERM, it will be - # terminated immediately after we cancel the job. However, the deeply - # spawned sleep will ignore it. We need to make sure that our - # implementation grants the sleep process a grace period and then - # kills it. - self.pre_run = [] - self.post_run = [] - self.parallel_cmd = os.path.join(fixtures.TEST_RESOURCES_CHECKS, - 'src', 'sleep_deeply.sh') - self.testjob._cancel_grace_period = 2 - self.prepare() - self.testjob.submit() - - # Stall a bit here to let the the spawned process start and install its - # signal handler for SIGTERM - time.sleep(1) - - t_grace = datetime.now() - self.testjob.cancel() - t_grace = datetime.now() - t_grace - self.testjob.wait() - - # Read pid of spawned sleep - with open(self.testjob.stdout) as f: - sleep_pid = int(f.read()) - - assert t_grace.total_seconds() >= 2 - assert self.testjob.state == 'TIMEOUT' - - # Verify that the spawned sleep is killed, too - self.assertProcessDied(sleep_pid) - - def test_guess_num_tasks(self): - # We want to trigger bug #1087 (Github), that's we set allocation +def assert_job_script_sanity(job): + '''Assert the sanity of the produced script file.''' + with open(job.script_filename) as fp: + matches = re.findall(r'echo prerun|echo postrun|hostname', + fp.read()) + assert ['echo prerun', 'hostname', 'echo postrun'] == matches + + +def _expected_slurm_directives(job): + return set([ + '#SBATCH --job-name="testjob"', + '#SBATCH --time=0:5:0', + '#SBATCH --output=%s' % job.stdout, + '#SBATCH --error=%s' % job.stderr, + '#SBATCH --ntasks=%s' % job.num_tasks, + '#SBATCH --ntasks-per-node=%s' % job.num_tasks_per_node, + '#SBATCH --ntasks-per-core=%s' % job.num_tasks_per_core, + '#SBATCH --ntasks-per-socket=%s' % job.num_tasks_per_socket, + '#SBATCH --cpus-per-task=%s' % job.num_cpus_per_task, + '#SBATCH --hint=multithread', + '#SBATCH --nodelist=%s' % job.sched_nodelist, + '#SBATCH --exclude=%s' % job.sched_exclude_nodelist, + '#SBATCH --partition=%s' % job.sched_partition, + '#SBATCH --reservation=%s' % job.sched_reservation, + '#SBATCH --account=%s' % job.sched_account, + '#SBATCH --exclusive', + # Custom options and directives + '#SBATCH --gres=gpu:4', + '#DW jobdw capacity=100GB', + '#DW stage_in source=/foo' + ]) + + +_expected_squeue_directives = _expected_slurm_directives + + +def _expected_pbs_directives(job): + num_nodes = job.num_tasks // job.num_tasks_per_node + num_cpus_per_node = job.num_cpus_per_task * job.num_tasks_per_node + return set([ + '#PBS -N "testjob"', + '#PBS -l walltime=0:5:0', + '#PBS -o %s' % job.stdout, + '#PBS -e %s' % job.stderr, + '#PBS -l select=%s:mpiprocs=%s:ncpus=%s' + ':mem=100GB:cpu_type=haswell' % (num_nodes, + job.num_tasks_per_node, + num_cpus_per_node), + '#PBS -q %s' % job.sched_partition, + '#PBS --gres=gpu:4', + '#DW jobdw capacity=100GB', + '#DW stage_in source=/foo' + ]) + + +def _expected_torque_directives(job): + num_nodes = job.num_tasks // job.num_tasks_per_node + num_cpus_per_node = job.num_cpus_per_task * job.num_tasks_per_node + return set([ + '#PBS -N "testjob"', + '#PBS -l walltime=0:5:0', + '#PBS -o %s' % job.stdout, + '#PBS -e %s' % job.stderr, + '#PBS -l nodes=%s:ppn=%s:haswell' % (num_nodes, num_cpus_per_node), + '#PBS -l mem=100GB', + '#PBS -q %s' % job.sched_partition, + '#PBS --gres=gpu:4', + '#DW jobdw capacity=100GB', + '#DW stage_in source=/foo' + ]) + + +def _expected_local_directives(job): + return set() + + +def test_prepare(fake_job): + sched_name = fake_job.scheduler.registered_name + if sched_name == 'pbs': + fake_job.options += ['mem=100GB', 'cpu_type=haswell'] + elif sched_name == 'torque': + fake_job.options += ['-l mem=100GB', 'haswell'] + + prepare_job(fake_job) + with open(fake_job.script_filename) as fp: + found_directives = set(re.findall(r'^\#\w+ .*', fp.read(), + re.MULTILINE)) + + expected_directives = globals()[f'_expected_{sched_name}_directives'] + assert_job_script_sanity(fake_job) + assert expected_directives(fake_job) == found_directives + + +def test_prepare_no_exclusive(make_job, slurm_only): + job = make_job(sched_exclusive_access=False) + prepare_job(job) + with open(job.script_filename) as fp: + assert re.search(r'--exclusive', fp.read()) is None + + +def test_prepare_no_smt(fake_job, slurm_only): + fake_job.use_smt = None + prepare_job(fake_job) + with open(fake_job.script_filename) as fp: + assert re.search(r'--hint', fp.read()) is None + + +def test_prepare_with_smt(fake_job, slurm_only): + fake_job.use_smt = True + prepare_job(fake_job) + with open(fake_job.script_filename) as fp: + assert re.search(r'--hint=multithread', fp.read()) is not None + + +def test_prepare_without_smt(fake_job, slurm_only): + fake_job.use_smt = False + prepare_job(fake_job) + with open(fake_job.script_filename) as fp: + assert re.search(r'--hint=nomultithread', fp.read()) is not None + + +def test_submit(make_job, exec_ctx): + minimal_job = make_job(sched_access=exec_ctx.access) + prepare_job(minimal_job) + assert minimal_job.nodelist is None + minimal_job.submit() + assert minimal_job.jobid is not None + minimal_job.wait() + + # Additional scheduler-specific checks + sched_name = minimal_job.scheduler.registered_name + if sched_name == 'local': + assert [socket.gethostname()] == minimal_job.nodelist + assert 0 == minimal_job.exitcode + elif sched_name == ('slurm', 'squeue'): + num_tasks_per_node = minimal_job.num_tasks_per_node or 1 + num_nodes = minimal_job.num_tasks // num_tasks_per_node + assert num_nodes == len(minimal_job.nodelist) + assert 0 == minimal_job.exitcode + + +def test_submit_timelimit(minimal_job, local_only): + minimal_job.time_limit = '2s' + prepare_job(minimal_job, 'sleep 10') + t_job = datetime.now() + minimal_job.submit() + assert minimal_job.jobid is not None + minimal_job.wait() + t_job = datetime.now() - t_job + assert t_job.total_seconds() >= 2 + assert t_job.total_seconds() < 3 + with open(minimal_job.stdout) as fp: + assert re.search('postrun', fp.read()) is None + + assert minimal_job.state == 'TIMEOUT' + + +def test_submit_job_array(make_job, slurm_only, exec_ctx): + job = make_job(sched_access=exec_ctx.access) + job.options = ['--array=0-1'] + prepare_job(job, command='echo "Task id: ${SLURM_ARRAY_TASK_ID}"') + job.submit() + job.wait() + assert job.exitcode == 0 + with open(job.stdout) as fp: + output = fp.read() + assert all([re.search('Task id: 0', output), + re.search('Task id: 1', output)]) + + +def test_cancel(make_job, exec_ctx): + minimal_job = make_job(sched_access=exec_ctx.access) + prepare_job(minimal_job, 'sleep 30') + t_job = datetime.now() + minimal_job.submit() + minimal_job.cancel() + minimal_job.wait() + t_job = datetime.now() - t_job + assert minimal_job.finished() + assert t_job.total_seconds() < 30 + + # Additional scheduler-specific checks + sched_name = minimal_job.scheduler.registered_name + if sched_name in ('slurm', 'squeue'): + assert minimal_job.state == 'CANCELLED' + + +def test_cancel_before_submit(minimal_job): + prepare_job(minimal_job, 'sleep 3') + with pytest.raises(JobNotStartedError): + minimal_job.cancel() + + +def test_wait_before_submit(minimal_job): + prepare_job(minimal_job, 'sleep 3') + with pytest.raises(JobNotStartedError): + minimal_job.wait() + + +def test_poll(make_job, exec_ctx): + minimal_job = make_job(sched_access=exec_ctx.access) + prepare_job(minimal_job, 'sleep 2') + minimal_job.submit() + assert not minimal_job.finished() + minimal_job.wait() + + +def test_poll_before_submit(minimal_job): + prepare_job(minimal_job, 'sleep 3') + with pytest.raises(JobNotStartedError): + minimal_job.finished() + + +def test_no_empty_lines_in_preamble(minimal_job): + for line in minimal_job.scheduler.emit_preamble(minimal_job): + assert line != '' + + +def test_guess_num_tasks(minimal_job, scheduler): + minimal_job.num_tasks = 0 + if scheduler.registered_name == 'local': + # We want to trigger bug #1087 (Github), that's why we set allocation # policy to idle. - self.testjob.num_tasks = 0 - self.testjob._sched_flex_alloc_nodes = 'idle' - self.prepare() - self.testjob.submit() - self.testjob.wait() - assert self.testjob.num_tasks == 1 - - def test_submit_max_pending_time(self): - pytest.skip('the maximum pending time has no effect on the ' - 'local scheduler') - - -class TestSlurmJob(_TestJob, unittest.TestCase): - @property - def sched_name(self): - return 'slurm' - - @property - def launcher_name(self): - return 'local' - - @property - def sched_configured(self): - return fixtures.partition_with_scheduler('slurm') is not None - - def setup_user(self, msg=None): - super().setup_user(msg='SLURM (with sacct) not configured') - - def _update_state(self, job): - job.state = 'PENDING' - - def test_prepare(self): - self.setup_job() - super().test_prepare() - expected_directives = set([ - '#SBATCH --job-name="testjob"', - '#SBATCH --time=0:5:0', - '#SBATCH --output=%s' % self.testjob.stdout, - '#SBATCH --error=%s' % self.testjob.stderr, - '#SBATCH --ntasks=%s' % self.testjob.num_tasks, - '#SBATCH --ntasks-per-node=%s' % self.testjob.num_tasks_per_node, - '#SBATCH --ntasks-per-core=%s' % self.testjob.num_tasks_per_core, - ('#SBATCH --ntasks-per-socket=%s' % - self.testjob.num_tasks_per_socket), - '#SBATCH --cpus-per-task=%s' % self.testjob.num_cpus_per_task, - '#SBATCH --hint=multithread', - '#SBATCH --nodelist=%s' % self.testjob.sched_nodelist, - '#SBATCH --exclude=%s' % self.testjob.sched_exclude_nodelist, - '#SBATCH --partition=%s' % self.testjob.sched_partition, - '#SBATCH --reservation=%s' % self.testjob.sched_reservation, - '#SBATCH --account=%s' % self.testjob.sched_account, - '#SBATCH --exclusive', - # Custom options and directives - '#SBATCH --gres=gpu:4', - '#DW jobdw capacity=100GB', - '#DW stage_in source=/foo' - ]) - with open(self.testjob.script_filename) as fp: - found_directives = set(re.findall(r'^\#\w+ .*', fp.read(), - re.MULTILINE)) - - assert expected_directives == found_directives - - def test_prepare_no_exclusive(self): - self.setup_job() - self.testjob._sched_exclusive_access = False - super().test_prepare() - with open(self.testjob.script_filename) as fp: - assert re.search(r'--exclusive', fp.read()) is None - - def test_prepare_no_smt(self): - self.setup_job() - self.testjob.use_smt = None - super().test_prepare() - with open(self.testjob.script_filename) as fp: - assert re.search(r'--hint', fp.read()) is None - - def test_prepare_with_smt(self): - self.setup_job() - self.testjob.use_smt = True - super().test_prepare() - with open(self.testjob.script_filename) as fp: - assert re.search(r'--hint=multithread', fp.read()) is not None - - def test_prepare_without_smt(self): - self.setup_job() - self.testjob.use_smt = False - super().test_prepare() - with open(self.testjob.script_filename) as fp: - assert re.search(r'--hint=nomultithread', fp.read()) is not None - - def test_submit(self): - super().test_submit() - assert 0 == self.testjob.exitcode - num_tasks_per_node = self.testjob.num_tasks_per_node or 1 - num_nodes = self.testjob.num_tasks // num_tasks_per_node - assert num_nodes == len(self.testjob.nodelist) - - def test_submit_timelimit(self): - # Skip this test for Slurm, since we the minimum time limit is 1min - pytest.skip("SLURM's minimum time limit is 60s") - - def test_cancel(self): - super().test_cancel() - assert self.testjob.state == 'CANCELLED' - - def test_guess_num_tasks(self): - self.testjob.num_tasks = 0 - self.testjob._sched_flex_alloc_nodes = 'all' + minimal_job.num_tasks = 0 + minimal_job._sched_flex_alloc_nodes = 'idle' + prepare_job(minimal_job) + minimal_job.submit() + minimal_job.wait() + assert minimal_job.num_tasks == 1 + elif scheduler.registered_name in ('slurm', 'squeue'): + minimal_job.num_tasks = 0 + minimal_job._sched_flex_alloc_nodes = 'all' # Monkey patch `allnodes()` to simulate extraction of # slurm nodes through the use of `scontrol show` - self.testjob.scheduler.allnodes = lambda: set() + minimal_job.scheduler.allnodes = lambda: set() # monkey patch `_get_default_partition()` to simulate extraction # of the default partition through the use of `scontrol show` - self.testjob.scheduler._get_default_partition = lambda: 'pdef' - assert self.testjob.guess_num_tasks() == 0 - - def test_submit_job_array(self): - self.testjob.options = ['--array=0-1'] - self.parallel_cmd = 'echo "Task id: ${SLURM_ARRAY_TASK_ID}"' - super().test_submit() - assert self.testjob.exitcode == 0 - with open(self.testjob.stdout) as fp: - output = fp.read() - assert all([re.search('Task id: 0', output), - re.search('Task id: 1', output)]) - - -class TestSqueueJob(TestSlurmJob): - @property - def sched_name(self): - return 'squeue' - - def setup_user(self, msg=None): - partition = (fixtures.partition_with_scheduler(self.sched_name) or - fixtures.partition_with_scheduler('slurm')) - if partition is None: - pytest.skip('SLURM not configured') - - self.testjob.options += partition.access - - def test_submit(self): - # Squeue backend may not set the exitcode; bypass our parent's submit - _TestJob.test_submit(self) - - -class TestPbsJob(_TestJob, unittest.TestCase): - @property - def sched_name(self): - return 'pbs' - - @property - def launcher_name(self): - return 'local' - - @property - def sched_configured(self): - return fixtures.partition_with_scheduler('pbs') is not None - - def setup_user(self, msg=None): - super().setup_user(msg='PBS not configured') - - @property - def testjob_options(self): - return self.testjob.options + ['mem=100GB', 'cpu_type=haswell'] - - @property - def node_select_options(self): - return [ - '#PBS -l select=%s:mpiprocs=%s:ncpus=%s' - ':mem=100GB:cpu_type=haswell' % ( - self.num_nodes, - self.testjob.num_tasks_per_node, - self.num_cpus_per_node - ) - ] - - @property - def expected_directives(self): - return set([ - '#PBS -N "testjob"', - '#PBS -l walltime=0:5:0', - '#PBS -o %s' % self.testjob.stdout, - '#PBS -e %s' % self.testjob.stderr, - *self.node_select_options, - '#PBS -q %s' % self.testjob.sched_partition, - '#PBS --gres=gpu:4', - '#DW jobdw capacity=100GB', - '#DW stage_in source=/foo' - ]) - - def test_prepare(self): - self.setup_job() - self.testjob.options = self.testjob_options - super().test_prepare() - self.num_nodes = (self.testjob.num_tasks // - self.testjob.num_tasks_per_node) - self.num_cpus_per_node = (self.testjob.num_cpus_per_task * - self.testjob.num_tasks_per_node) - with open(self.testjob.script_filename) as fp: - found_directives = set(re.findall(r'^\#\w+ .*', fp.read(), - re.MULTILINE)) - - assert self.expected_directives == found_directives - - def test_prepare_no_cpus(self): - self.setup_job() - self.testjob.num_cpus_per_task = None - self.testjob.options = self.testjob_options - super().test_prepare() - self.num_nodes = (self.testjob.num_tasks // - self.testjob.num_tasks_per_node) - self.num_cpus_per_node = self.testjob.num_tasks_per_node - with open(self.testjob.script_filename) as fp: - found_directives = set(re.findall(r'^\#\w+ .*', fp.read(), - re.MULTILINE)) - - assert self.expected_directives == found_directives - - def test_submit_timelimit(self): - # Skip this test for PBS, since the minimum time limit is 1min - pytest.skip("PBS minimum time limit is 60s") - - def test_submit_max_pending_time(self): - pytest.skip('not implemented for the pbs scheduler') - - -class TestTorqueJob(TestPbsJob): - @property - def sched_name(self): - return 'torque' - - @property - def sched_configured(self): - return fixtures.partition_with_scheduler('torque') is not None - - def setup_user(self, msg=None): - super().setup_user(msg='Torque not configured') - - @property - def testjob_options(self): - return self.testjob.options + ['-l mem=100GB', 'haswell'] - - @property - def node_select_options(self): - return [ - '#PBS -l nodes=%s:ppn=%s:haswell' % (self.num_nodes, - self.num_cpus_per_node), - '#PBS -l mem=100GB' - ] - - def test_submit_timelimit(self): - # Skip this test for PBS, since we the minimum time limit is 1min - pytest.skip("Torque minimum time limit is 60s") - - def _update_state(self, job): - job.state = 'QUEUED' - - def test_submit_max_pending_time(self): - _TestJob.test_submit_max_pending_time(self) - - -class TestSlurmFlexibleNodeAllocation(unittest.TestCase): - def create_dummy_nodes(obj): - node_descriptions = ['NodeName=nid00001 Arch=x86_64 CoresPerSocket=12 ' - 'CPUAlloc=0 CPUErr=0 CPUTot=24 CPULoad=0.00 ' - 'AvailableFeatures=f1,f2 ActiveFeatures=f1,f2 ' - 'Gres=gpu_mem:16280,gpu:1 NodeAddr=nid00001 ' - 'NodeHostName=nid00001 Version=10.00 OS=Linux ' - 'RealMemory=32220 AllocMem=0 FreeMem=10000 ' - 'Sockets=1 Boards=1 State=MAINT+DRAIN ' - 'ThreadsPerCore=2 TmpDisk=0 Weight=1 Owner=N/A ' - 'MCS_label=N/A Partitions=p1,p2,pdef ' - 'BootTime=01 Jan 2018 ' - 'SlurmdStartTime=01 Jan 2018 ' - 'CfgTRES=cpu=24,mem=32220M ' - 'AllocTRES= CapWatts=n/a CurrentWatts=100 ' - 'LowestJoules=100000000 ConsumedJoules=0 ' - 'ExtSensorsJoules=n/s ExtSensorsWatts=0 ' - 'ExtSensorsTemp=n/s Reason=Foo/ ' - 'failed [reframe_user@01 Jan 2018]', - - 'NodeName=nid00002 Arch=x86_64 CoresPerSocket=12 ' - 'CPUAlloc=0 CPUErr=0 CPUTot=24 CPULoad=0.00 ' - 'AvailableFeatures=f2,f3 ActiveFeatures=f2,f3 ' - 'Gres=gpu_mem:16280,gpu:1 NodeAddr=nid00002 ' - 'NodeHostName=nid00002 Version=10.00 OS=Linux ' - 'RealMemory=32220 AllocMem=0 FreeMem=10000 ' - 'Sockets=1 Boards=1 State=MAINT+DRAIN ' - 'ThreadsPerCore=2 TmpDisk=0 Weight=1 Owner=N/A ' - 'MCS_label=N/A Partitions=p2,p3,pdef ' - 'BootTime=01 Jan 2018 ' - 'SlurmdStartTime=01 Jan 2018 ' - 'CfgTRES=cpu=24,mem=32220M ' - 'AllocTRES= CapWatts=n/a CurrentWatts=100 ' - 'LowestJoules=100000000 ConsumedJoules=0 ' - 'ExtSensorsJoules=n/s ExtSensorsWatts=0 ' - 'ExtSensorsTemp=n/s Reason=Foo/ ' - 'failed [reframe_user@01 Jan 2018]', - - 'Node invalid_node1 not found', - - 'NodeName=nid00003 Arch=x86_64 CoresPerSocket=12 ' - 'CPUAlloc=0 CPUErr=0 CPUTot=24 CPULoad=0.00 ' - 'AvailableFeatures=f1,f3 ActiveFeatures=f1,f3 ' - 'Gres=gpu_mem:16280,gpu:1 NodeAddr=nid00003' - 'NodeHostName=nid00003 Version=10.00 OS=Linux ' - 'RealMemory=32220 AllocMem=0 FreeMem=10000 ' - 'Sockets=1 Boards=1 State=IDLE ' - 'ThreadsPerCore=2 TmpDisk=0 Weight=1 Owner=N/A ' - 'MCS_label=N/A Partitions=p1,p3,pdef ' - 'BootTime=01 Jan 2018 ' - 'SlurmdStartTime=01 Jan 2018 ' - 'CfgTRES=cpu=24,mem=32220M ' - 'AllocTRES= CapWatts=n/a CurrentWatts=100 ' - 'LowestJoules=100000000 ConsumedJoules=0 ' - 'ExtSensorsJoules=n/s ExtSensorsWatts=0 ' - 'ExtSensorsTemp=n/s Reason=Foo/ ' - 'failed [reframe_user@01 Jan 2018]', - - 'NodeName=nid00004 Arch=x86_64 CoresPerSocket=12 ' - 'CPUAlloc=0 CPUErr=0 CPUTot=24 CPULoad=0.00 ' - 'AvailableFeatures=f1,f4 ActiveFeatures=f1,f4 ' - 'Gres=gpu_mem:16280,gpu:1 NodeAddr=nid00003' - 'NodeHostName=nid00003 Version=10.00 OS=Linux ' - 'RealMemory=32220 AllocMem=0 FreeMem=10000 ' - 'Sockets=1 Boards=1 State=IDLE ' - 'ThreadsPerCore=2 TmpDisk=0 Weight=1 Owner=N/A ' - 'MCS_label=N/A Partitions=p1,p3,pdef ' - 'BootTime=01 Jan 2018 ' - 'SlurmdStartTime=01 Jan 2018 ' - 'CfgTRES=cpu=24,mem=32220M ' - 'AllocTRES= CapWatts=n/a CurrentWatts=100 ' - 'LowestJoules=100000000 ConsumedJoules=0 ' - 'ExtSensorsJoules=n/s ExtSensorsWatts=0 ' - 'ExtSensorsTemp=n/s Reason=Foo/ ', - - 'NodeName=nid00005 Arch=x86_64 CoresPerSocket=12 ' - 'CPUAlloc=0 CPUErr=0 CPUTot=24 CPULoad=0.00 ' - 'AvailableFeatures=f5 ActiveFeatures=f5 ' - 'Gres=gpu_mem:16280,gpu:1 NodeAddr=nid00003' - 'NodeHostName=nid00003 Version=10.00 OS=Linux ' - 'RealMemory=32220 AllocMem=0 FreeMem=10000 ' - 'Sockets=1 Boards=1 State=ALLOCATED ' - 'ThreadsPerCore=2 TmpDisk=0 Weight=1 Owner=N/A ' - 'MCS_label=N/A Partitions=p1,p3 ' - 'BootTime=01 Jan 2018 ' - 'SlurmdStartTime=01 Jan 2018 ' - 'CfgTRES=cpu=24,mem=32220M ' - 'AllocTRES= CapWatts=n/a CurrentWatts=100 ' - 'LowestJoules=100000000 ConsumedJoules=0 ' - 'ExtSensorsJoules=n/s ExtSensorsWatts=0 ' - 'ExtSensorsTemp=n/s Reason=Foo/ ' - 'failed [reframe_user@01 Jan 2018]', - - 'Node invalid_node2 not found'] - - return _create_nodes(node_descriptions) - - def create_reservation_nodes(self, res): - return {n for n in self.testjob.scheduler.allnodes() - if n.name != 'nid00001'} - - def create_dummy_nodes_by_name(self, name): - return {n for n in self.testjob.scheduler.allnodes() if n.name == name} + minimal_job.scheduler._get_default_partition = lambda: 'pdef' + assert minimal_job.guess_num_tasks() == 0 + else: + with pytest.raises(NotImplementedError): + minimal_job.guess_num_tasks() - def setUp(self): - # Monkey patch scheduler to simulate retrieval of nodes from Slurm - patched_sched = getscheduler('slurm')() - patched_sched.allnodes = self.create_dummy_nodes - patched_sched._get_default_partition = lambda: 'pdef' - - self.workdir = tempfile.mkdtemp(dir='unittests') - self.testjob = Job.create( - patched_sched, getlauncher('local')(), - name='testjob', - workdir=self.workdir, - script_filename=os.path.join(self.workdir, 'testjob.sh'), - stdout=os.path.join(self.workdir, 'testjob.out'), - stderr=os.path.join(self.workdir, 'testjob.err') - ) - self.testjob._sched_flex_alloc_nodes = 'all' - self.testjob.num_tasks_per_node = 4 - self.testjob.num_tasks = 0 - - def tearDown(self): - os_ext.rmtree(self.workdir) - - def test_positive_flex_alloc_nodes(self): - self.testjob._sched_flex_alloc_nodes = 12 - self.testjob._sched_access = ['--constraint=f1'] - self.prepare_job() - assert self.testjob.num_tasks == 48 - - def test_zero_flex_alloc_nodes(self): - self.testjob._sched_flex_alloc_nodes = 0 - self.testjob._sched_access = ['--constraint=f1'] - with pytest.raises(JobError): - self.prepare_job() - - def test_negative_flex_alloc_nodes(self): - self.testjob._sched_flex_alloc_nodes = -1 - self.testjob._sched_access = ['--constraint=f1'] - with pytest.raises(JobError): - self.prepare_job() - - def test_sched_access_idle(self): - self.testjob._sched_flex_alloc_nodes = 'idle' - self.testjob._sched_access = ['--constraint=f1'] - self.prepare_job() - assert self.testjob.num_tasks == 8 - - def test_sched_access_idle_sequence_view(self): - from reframe.utility import SequenceView - - self.testjob._sched_flex_alloc_nodes = 'idle' - - # Here simulate passing a readonly 'sched_access' as returned - # by a 'SystemPartition' instance. - self.testjob._sched_access = SequenceView(['--constraint=f3']) - self.testjob._sched_partition = 'p3' - self.prepare_job() - assert self.testjob.num_tasks == 4 - - def test_sched_access_constraint_partition(self): - self.testjob._sched_flex_alloc_nodes = 'all' - self.testjob._sched_access = ['--constraint=f1', '--partition=p2'] - self.prepare_job() - assert self.testjob.num_tasks == 4 - - def test_sched_access_partition(self): - self.testjob._sched_access = ['--partition=p1'] - self.prepare_job() - assert self.testjob.num_tasks == 16 - - def test_default_partition_all(self): - self.testjob._sched_flex_alloc_nodes = 'all' - self.prepare_job() - assert self.testjob.num_tasks == 16 - - def test_constraint_idle(self): - self.testjob._sched_flex_alloc_nodes = 'idle' - self.testjob.options = ['--constraint=f1'] - self.prepare_job() - assert self.testjob.num_tasks == 8 - - def test_partition_idle(self): - self.testjob._sched_flex_alloc_nodes = 'idle' - self.testjob._sched_partition = 'p2' - with pytest.raises(JobError): - self.prepare_job() - - def test_valid_constraint_opt(self): - self.testjob.options = ['-C f1'] - self.prepare_job() - assert self.testjob.num_tasks == 12 - - def test_valid_multiple_constraints(self): - self.testjob.options = ['-C f1,f3'] - self.prepare_job() - assert self.testjob.num_tasks == 4 - - def test_valid_partition_cmd(self): - self.testjob._sched_partition = 'p2' - self.prepare_job() - assert self.testjob.num_tasks == 8 - - def test_valid_partition_opt(self): - self.testjob.options = ['-p p2'] - self.prepare_job() - assert self.testjob.num_tasks == 8 - - def test_valid_multiple_partitions(self): - self.testjob.options = ['--partition=p1,p2'] - self.prepare_job() - assert self.testjob.num_tasks == 4 - - def test_valid_constraint_partition(self): - self.testjob.options = ['-C f1,f2', '--partition=p1,p2'] - self.prepare_job() - assert self.testjob.num_tasks == 4 - - def test_not_valid_partition_cmd(self): - self.testjob._sched_partition = 'invalid' - with pytest.raises(JobError): - self.prepare_job() - - def test_invalid_partition_opt(self): - self.testjob.options = ['--partition=invalid'] - with pytest.raises(JobError): - self.prepare_job() - - def test_invalid_constraint(self): - self.testjob.options = ['--constraint=invalid'] - with pytest.raises(JobError): - self.prepare_job() - - def test_valid_reservation_cmd(self): - self.testjob._sched_access = ['--constraint=f2'] - self.testjob._sched_reservation = 'dummy' - - # Monkey patch `_get_reservation_nodes` to simulate extraction of - # reservation slurm nodes through the use of `scontrol show` - sched = self.testjob.scheduler - sched._get_reservation_nodes = self.create_reservation_nodes - self.prepare_job() - assert self.testjob.num_tasks == 4 - - def test_valid_reservation_option(self): - self.testjob._sched_access = ['--constraint=f2'] - self.testjob.options = ['--reservation=dummy'] - sched = self.testjob.scheduler - sched._get_reservation_nodes = self.create_reservation_nodes - self.prepare_job() - assert self.testjob.num_tasks == 4 - - def test_exclude_nodes_cmd(self): - self.testjob._sched_access = ['--constraint=f1'] - self.testjob._sched_exclude_nodelist = 'nid00001' - - # Monkey patch `_get_nodes_by_name` to simulate extraction of - # slurm nodes by name through the use of `scontrol show` - sched = self.testjob.scheduler - sched._get_nodes_by_name = self.create_dummy_nodes_by_name - self.prepare_job() - assert self.testjob.num_tasks == 8 - - def test_exclude_nodes_opt(self): - self.testjob._sched_access = ['--constraint=f1'] - self.testjob.options = ['-x nid00001'] - sched = self.testjob.scheduler - sched._get_nodes_by_name = self.create_dummy_nodes_by_name - self.prepare_job() - assert self.testjob.num_tasks == 8 - - def test_no_num_tasks_per_node(self): - self.testjob.num_tasks_per_node = None - self.testjob.options = ['-C f1,f2', '--partition=p1,p2'] - self.prepare_job() - assert self.testjob.num_tasks == 1 - - def test_not_enough_idle_nodes(self): - self.testjob._sched_flex_alloc_nodes = 'idle' - self.testjob.num_tasks = -12 - with pytest.raises(JobError): - self.prepare_job() - - def test_not_enough_nodes_constraint_partition(self): - self.testjob.options = ['-C f1,f2', '--partition=p1,p2'] - self.testjob.num_tasks = -8 - with pytest.raises(JobError): - self.prepare_job() - - def test_enough_nodes_constraint_partition(self): - self.testjob.options = ['-C f1,f2', '--partition=p1,p2'] - self.testjob.num_tasks = -4 - self.prepare_job() - assert self.testjob.num_tasks == 4 - - def prepare_job(self): - self.testjob.prepare(['hostname']) - - -class TestSlurmNode(unittest.TestCase): - def setUp(self): - allocated_node_description = ( - 'NodeName=nid00001 Arch=x86_64 CoresPerSocket=12 ' + +def test_submit_max_pending_time(make_job, exec_ctx, scheduler): + if scheduler.registered_name in ('local', 'pbs'): + pytest.skip(f"max_pending_time not supported by the " + f"'{scheduler.registered_name}' scheduler") + + def update_state(job): + if scheduler.registered_name in ('slurm', 'squeue'): + job.state = 'PENDING' + elif scheduler.registered_name == 'torque': + job.state = 'QUEUED' + else: + # This should not happen + assert 0 + + minimal_job = make_job(sched_access=exec_ctx.access) + prepare_job(minimal_job, 'sleep 30') + + # Monkey patch `self._update_state` to simulate that the job is + # pending on the queue for enough time so it can be canceled due + # to exceeding the maximum pending time + minimal_job.scheduler._update_state = update_state + minimal_job._max_pending_time = timedelta(milliseconds=50) + minimal_job.submit() + with pytest.raises(JobError, + match='maximum pending time exceeded'): + minimal_job.wait() + + +def assert_process_died(pid): + try: + os.kill(pid, 0) + pytest.fail('process %s is still alive' % pid) + except (ProcessLookupError, PermissionError): + pass + + +def test_cancel_with_grace(minimal_job, scheduler, local_only): + # This test emulates a spawned process that ignores the SIGTERM signal + # and also spawns another process: + # + # reframe --- local job script --- sleep 10 + # (TERM IGN) + # + # We expect the job not to be cancelled immediately, since it ignores + # the gracious signal we are sending it. However, we expect it to be + # killed immediately after the grace period of 2 seconds expires. + # + # We also check that the additional spawned process is also killed. + minimal_job.time_limit = '1m' + minimal_job.scheduler._cancel_grace_period = 2 + prepare_job(minimal_job, + command='sleep 5 &', + pre_run=['trap -- "" TERM'], + post_run=['echo $!', 'wait']) + minimal_job.submit() + + # Stall a bit here to let the the spawned process start and install its + # signal handler for SIGTERM + time.sleep(1) + + t_grace = datetime.now() + minimal_job.cancel() + t_grace = datetime.now() - t_grace + + minimal_job.wait() + # Read pid of spawned sleep + with open(minimal_job.stdout) as fp: + sleep_pid = int(fp.read()) + + assert t_grace.total_seconds() >= 2 + assert t_grace.total_seconds() < 5 + assert minimal_job.state == 'TIMEOUT' + + # Verify that the spawned sleep is killed, too + assert_process_died(sleep_pid) + + +def test_cancel_term_ignore(minimal_job, scheduler, local_only): + # This test emulates a descendant process of the spawned job that + # ignores the SIGTERM signal: + # + # reframe --- local job script --- sleep_deeply.sh --- sleep + # (TERM IGN) + # + # Since the "local job script" does not ignore SIGTERM, it will be + # terminated immediately after we cancel the job. However, the deeply + # spawned sleep will ignore it. We need to make sure that our + # implementation grants the sleep process a grace period and then + # kills it. + minimal_job.time_limit = '1m' + minimal_job.scheduler._cancel_grace_period = 2 + prepare_job(minimal_job, + command=os.path.join(fixtures.TEST_RESOURCES_CHECKS, + 'src', 'sleep_deeply.sh'), + pre_run=[''], + post_run=['']) + minimal_job.submit() + + # Stall a bit here to let the the spawned process start and install its + # signal handler for SIGTERM + time.sleep(1) + + t_grace = datetime.now() + minimal_job.cancel() + t_grace = datetime.now() - t_grace + minimal_job.wait() + + # Read pid of spawned sleep + with open(minimal_job.stdout) as fp: + sleep_pid = int(fp.read()) + + assert t_grace.total_seconds() >= 2 + assert minimal_job.state == 'TIMEOUT' + + # Verify that the spawned sleep is killed, too + assert_process_died(sleep_pid) + + +# Flexible node allocation tests + + +@pytest.fixture +def slurm_nodes(): + '''Dummy Slurm node descriptions''' + return ['NodeName=nid00001 Arch=x86_64 CoresPerSocket=12 ' 'CPUAlloc=0 CPUErr=0 CPUTot=24 CPULoad=0.00 ' 'AvailableFeatures=f1,f2 ActiveFeatures=f1,f2 ' 'Gres=gpu_mem:16280,gpu:1 NodeAddr=nid00001 ' 'NodeHostName=nid00001 Version=10.00 OS=Linux ' 'RealMemory=32220 AllocMem=0 FreeMem=10000 ' - 'Sockets=1 Boards=1 State=ALLOCATED ' + 'Sockets=1 Boards=1 State=MAINT+DRAIN ' 'ThreadsPerCore=2 TmpDisk=0 Weight=1 Owner=N/A ' - 'MCS_label=N/A Partitions=p1,p2 ' + 'MCS_label=N/A Partitions=p1,p2,pdef ' 'BootTime=01 Jan 2018 ' 'SlurmdStartTime=01 Jan 2018 ' 'CfgTRES=cpu=24,mem=32220M ' @@ -913,19 +526,17 @@ def setUp(self): 'LowestJoules=100000000 ConsumedJoules=0 ' 'ExtSensorsJoules=n/s ExtSensorsWatts=0 ' 'ExtSensorsTemp=n/s Reason=Foo/ ' - 'failed [reframe_user@01 Jan 2018]' - ) + 'failed [reframe_user@01 Jan 2018]', - idle_node_description = ( 'NodeName=nid00002 Arch=x86_64 CoresPerSocket=12 ' 'CPUAlloc=0 CPUErr=0 CPUTot=24 CPULoad=0.00 ' - 'AvailableFeatures=f1,f2 ActiveFeatures=f1,f2 ' - 'Gres=gpu_mem:16280,gpu:1 NodeAddr=nid00001 ' - 'NodeHostName=nid00001 Version=10.00 OS=Linux ' + 'AvailableFeatures=f2,f3 ActiveFeatures=f2,f3 ' + 'Gres=gpu_mem:16280,gpu:1 NodeAddr=nid00002 ' + 'NodeHostName=nid00002 Version=10.00 OS=Linux ' 'RealMemory=32220 AllocMem=0 FreeMem=10000 ' - 'Sockets=1 Boards=1 State=IDLE ' + 'Sockets=1 Boards=1 State=MAINT+DRAIN ' 'ThreadsPerCore=2 TmpDisk=0 Weight=1 Owner=N/A ' - 'MCS_label=N/A Partitions=p1,p2 ' + 'MCS_label=N/A Partitions=p2,p3,pdef ' 'BootTime=01 Jan 2018 ' 'SlurmdStartTime=01 Jan 2018 ' 'CfgTRES=cpu=24,mem=32220M ' @@ -933,19 +544,19 @@ def setUp(self): 'LowestJoules=100000000 ConsumedJoules=0 ' 'ExtSensorsJoules=n/s ExtSensorsWatts=0 ' 'ExtSensorsTemp=n/s Reason=Foo/ ' - 'failed [reframe_user@01 Jan 2018]' - ) + 'failed [reframe_user@01 Jan 2018]', + + 'Node invalid_node1 not found', - idle_drained_node_description = ( 'NodeName=nid00003 Arch=x86_64 CoresPerSocket=12 ' 'CPUAlloc=0 CPUErr=0 CPUTot=24 CPULoad=0.00 ' - 'AvailableFeatures=f1,f2 ActiveFeatures=f1,f2 ' - 'Gres=gpu_mem:16280,gpu:1 NodeAddr=nid00001 ' - 'NodeHostName=nid00001 Version=10.00 OS=Linux ' + 'AvailableFeatures=f1,f3 ActiveFeatures=f1,f3 ' + 'Gres=gpu_mem:16280,gpu:1 NodeAddr=nid00003' + 'NodeHostName=nid00003 Version=10.00 OS=Linux ' 'RealMemory=32220 AllocMem=0 FreeMem=10000 ' - 'Sockets=1 Boards=1 State=IDLE+DRAIN ' + 'Sockets=1 Boards=1 State=IDLE ' 'ThreadsPerCore=2 TmpDisk=0 Weight=1 Owner=N/A ' - 'MCS_label=N/A Partitions=p1,p2 ' + 'MCS_label=N/A Partitions=p1,p3,pdef ' 'BootTime=01 Jan 2018 ' 'SlurmdStartTime=01 Jan 2018 ' 'CfgTRES=cpu=24,mem=32220M ' @@ -953,29 +564,367 @@ def setUp(self): 'LowestJoules=100000000 ConsumedJoules=0 ' 'ExtSensorsJoules=n/s ExtSensorsWatts=0 ' 'ExtSensorsTemp=n/s Reason=Foo/ ' - 'failed [reframe_user@01 Jan 2018]' - ) + 'failed [reframe_user@01 Jan 2018]', - no_partition_node_description = ( 'NodeName=nid00004 Arch=x86_64 CoresPerSocket=12 ' 'CPUAlloc=0 CPUErr=0 CPUTot=24 CPULoad=0.00 ' - 'AvailableFeatures=f1,f2 ActiveFeatures=f1,f2 ' - 'Gres=gpu_mem:16280,gpu:1 NodeAddr=nid00001 ' - 'NodeHostName=nid00001 Version=10.00 OS=Linux ' + 'AvailableFeatures=f1,f4 ActiveFeatures=f1,f4 ' + 'Gres=gpu_mem:16280,gpu:1 NodeAddr=nid00004' + 'NodeHostName=nid00004 Version=10.00 OS=Linux ' 'RealMemory=32220 AllocMem=0 FreeMem=10000 ' - 'Sockets=1 Boards=1 State=IDLE+DRAIN ' + 'Sockets=1 Boards=1 State=IDLE ' 'ThreadsPerCore=2 TmpDisk=0 Weight=1 Owner=N/A ' - 'MCS_label=N/A BootTime=01 Jan 2018 ' + 'MCS_label=N/A Partitions=p1,p3,pdef ' + 'BootTime=01 Jan 2018 ' + 'SlurmdStartTime=01 Jan 2018 ' + 'CfgTRES=cpu=24,mem=32220M ' + 'AllocTRES= CapWatts=n/a CurrentWatts=100 ' + 'LowestJoules=100000000 ConsumedJoules=0 ' + 'ExtSensorsJoules=n/s ExtSensorsWatts=0 ' + 'ExtSensorsTemp=n/s Reason=Foo/ ', + + 'NodeName=nid00005 Arch=x86_64 CoresPerSocket=12 ' + 'CPUAlloc=0 CPUErr=0 CPUTot=24 CPULoad=0.00 ' + 'AvailableFeatures=f5 ActiveFeatures=f5 ' + 'Gres=gpu_mem:16280,gpu:1 NodeAddr=nid00005' + 'NodeHostName=nid00005 Version=10.00 OS=Linux ' + 'RealMemory=32220 AllocMem=0 FreeMem=10000 ' + 'Sockets=1 Boards=1 State=ALLOCATED ' + 'ThreadsPerCore=2 TmpDisk=0 Weight=1 Owner=N/A ' + 'MCS_label=N/A Partitions=p1,p3 ' + 'BootTime=01 Jan 2018 ' 'SlurmdStartTime=01 Jan 2018 ' 'CfgTRES=cpu=24,mem=32220M ' 'AllocTRES= CapWatts=n/a CurrentWatts=100 ' 'LowestJoules=100000000 ConsumedJoules=0 ' 'ExtSensorsJoules=n/s ExtSensorsWatts=0 ' 'ExtSensorsTemp=n/s Reason=Foo/ ' - 'failed [reframe_user@01 Jan 2018]' + 'failed [reframe_user@01 Jan 2018]', + + 'Node invalid_node2 not found'] + + +@pytest.fixture +def slurm_scheduler_patched(slurm_nodes): + ret = getscheduler('slurm')() + ret.allnodes = lambda: _create_nodes(slurm_nodes) + ret._get_default_partition = lambda: 'pdef' + ret._get_reservation_nodes = lambda res: { + n for n in ret.allnodes() if n.name != 'nid00001' + } + ret._get_nodes_by_name = lambda name: { + n for n in ret.allnodes() if n.name == name + } + return ret + + +@pytest.fixture +def make_flexible_job(slurm_scheduler_patched, tmp_path): + def _make_flexible_job(flex_type, **jobargs): + ret = Job.create( + slurm_scheduler_patched, getlauncher('local')(), + name='testjob', + workdir=tmp_path, + script_filename=str(tmp_path / 'job.sh'), + stdout=str(tmp_path / 'job.out'), + stderr=str(tmp_path / 'job.err'), + sched_flex_alloc_nodes=flex_type, + **jobargs ) + ret.num_tasks = 0 + ret.num_tasks_per_node = 4 + return ret - self.no_name_node_description = ( + return _make_flexible_job + + +def test_flex_alloc_nodes_positive(make_flexible_job): + job = make_flexible_job(12, sched_access=['--constraint=f1']) + prepare_job(job) + assert job.num_tasks == 48 + + +def test_flex_alloc_nodes_zero(make_flexible_job): + job = make_flexible_job(0, sched_access=['--constraint=f1']) + with pytest.raises(JobError): + prepare_job(job) + + +def test_flex_alloc_nodes_negative(make_flexible_job): + job = make_flexible_job(-1, sched_access=['--constraint=f1']) + with pytest.raises(JobError): + prepare_job(job) + + +def test_flex_alloc_sched_access_idle(make_flexible_job): + job = make_flexible_job('idle', sched_access=['--constraint=f1']) + prepare_job(job) + assert job.num_tasks == 8 + + +def test_flex_alloc_sched_access_idle_sequence_view(make_flexible_job): + # Here we simulate passing a readonly 'sched_access' as returned + # by a 'SystemPartition' instance. + + from reframe.utility import SequenceView + + job = make_flexible_job('idle', + sched_access=SequenceView(['--constraint=f3']), + sched_partition='p3') + prepare_job(job) + assert job.num_tasks == 4 + + +def test_flex_alloc_sched_access_constraint_partition(make_flexible_job): + job = make_flexible_job( + 'all', sched_access=['--constraint=f1', '--partition=p2'] + ) + prepare_job(job) + assert job.num_tasks == 4 + + +def test_flex_alloc_sched_access_partition(make_flexible_job): + job = make_flexible_job('all', sched_access=['--partition=p1']) + prepare_job(job) + assert job.num_tasks == 16 + + +def test_flex_alloc_default_partition_all(make_flexible_job): + job = make_flexible_job('all') + prepare_job(job) + assert job.num_tasks == 16 + + +def test_flex_alloc_constraint_idle(make_flexible_job): + job = make_flexible_job('idle') + job.options = ['--constraint=f1'] + prepare_job(job) + assert job.num_tasks == 8 + + +def test_flex_alloc_partition_idle(make_flexible_job): + job = make_flexible_job('idle', sched_partition='p2') + with pytest.raises(JobError): + prepare_job(job) + + +def test_flex_alloc_valid_constraint_opt(make_flexible_job): + job = make_flexible_job('all') + job.options = ['-C f1'] + prepare_job(job) + assert job.num_tasks == 12 + + +def test_flex_alloc_valid_multiple_constraints(make_flexible_job): + job = make_flexible_job('all') + job.options = ['-C f1,f3'] + prepare_job(job) + assert job.num_tasks == 4 + + +def test_flex_alloc_valid_partition_cmd(make_flexible_job): + job = make_flexible_job('all', sched_partition='p2') + prepare_job(job) + assert job.num_tasks == 8 + + +def test_flex_alloc_valid_partition_opt(make_flexible_job): + job = make_flexible_job('all') + job.options = ['-p p2'] + prepare_job(job) + assert job.num_tasks == 8 + + +def test_flex_alloc_valid_multiple_partitions(make_flexible_job): + job = make_flexible_job('all') + job.options = ['--partition=p1,p2'] + prepare_job(job) + assert job.num_tasks == 4 + + +def test_flex_alloc_valid_constraint_partition(make_flexible_job): + job = make_flexible_job('all') + job.options = ['-C f1,f2', '--partition=p1,p2'] + prepare_job(job) + assert job.num_tasks == 4 + + +def test_flex_alloc_invalid_partition_cmd(make_flexible_job): + job = make_flexible_job('all', sched_partition='invalid') + with pytest.raises(JobError): + prepare_job(job) + + +def test_flex_alloc_invalid_partition_opt(make_flexible_job): + job = make_flexible_job('all') + job.options = ['--partition=invalid'] + with pytest.raises(JobError): + prepare_job(job) + + +def test_flex_alloc_invalid_constraint(make_flexible_job): + job = make_flexible_job('all') + job.options = ['--constraint=invalid'] + with pytest.raises(JobError): + prepare_job(job) + + +def test_flex_alloc_valid_reservation_cmd(make_flexible_job): + job = make_flexible_job('all', + sched_access=['--constraint=f2'], + sched_reservation='dummy') + + prepare_job(job) + assert job.num_tasks == 4 + + +def test_flex_alloc_valid_reservation_option(make_flexible_job): + job = make_flexible_job('all', sched_access=['--constraint=f2']) + job.options = ['--reservation=dummy'] + prepare_job(job) + assert job.num_tasks == 4 + + +def test_flex_alloc_exclude_nodes_cmd(make_flexible_job): + job = make_flexible_job('all', + sched_access=['--constraint=f1'], + sched_exclude_nodelist='nid00001') + prepare_job(job) + assert job.num_tasks == 8 + + +def test_flex_alloc_exclude_nodes_opt(make_flexible_job): + job = make_flexible_job('all', sched_access=['--constraint=f1']) + job.options = ['-x nid00001'] + prepare_job(job) + assert job.num_tasks == 8 + + +def test_flex_alloc_no_num_tasks_per_node(make_flexible_job): + job = make_flexible_job('all') + job.num_tasks_per_node = None + job.options = ['-C f1,f2', '--partition=p1,p2'] + prepare_job(job) + assert job.num_tasks == 1 + + +def test_flex_alloc_not_enough_idle_nodes(make_flexible_job): + job = make_flexible_job('idle') + job.num_tasks = -12 + with pytest.raises(JobError): + prepare_job(job) + + +def test_flex_alloc_not_enough_nodes_constraint_partition(make_flexible_job): + job = make_flexible_job('all') + job.options = ['-C f1,f2', '--partition=p1,p2'] + job.num_tasks = -8 + with pytest.raises(JobError): + prepare_job(job) + + +def test_flex_alloc_enough_nodes_constraint_partition(make_flexible_job): + job = make_flexible_job('all') + job.options = ['-C f1,f2', '--partition=p1,p2'] + job.num_tasks = -4 + prepare_job(job) + assert job.num_tasks == 4 + + +@pytest.fixture +def slurm_node_allocated(): + return _SlurmNode( + 'NodeName=nid00001 Arch=x86_64 CoresPerSocket=12 ' + 'CPUAlloc=0 CPUErr=0 CPUTot=24 CPULoad=0.00 ' + 'AvailableFeatures=f1,f2 ActiveFeatures=f1,f2 ' + 'Gres=gpu_mem:16280,gpu:1 NodeAddr=nid00001 ' + 'NodeHostName=nid00001 Version=10.00 OS=Linux ' + 'RealMemory=32220 AllocMem=0 FreeMem=10000 ' + 'Sockets=1 Boards=1 State=ALLOCATED ' + 'ThreadsPerCore=2 TmpDisk=0 Weight=1 Owner=N/A ' + 'MCS_label=N/A Partitions=p1,p2 ' + 'BootTime=01 Jan 2018 ' + 'SlurmdStartTime=01 Jan 2018 ' + 'CfgTRES=cpu=24,mem=32220M ' + 'AllocTRES= CapWatts=n/a CurrentWatts=100 ' + 'LowestJoules=100000000 ConsumedJoules=0 ' + 'ExtSensorsJoules=n/s ExtSensorsWatts=0 ' + 'ExtSensorsTemp=n/s Reason=Foo/ ' + 'failed [reframe_user@01 Jan 2018]' + ) + + +@pytest.fixture +def slurm_node_idle(): + return _SlurmNode( + 'NodeName=nid00002 Arch=x86_64 CoresPerSocket=12 ' + 'CPUAlloc=0 CPUErr=0 CPUTot=24 CPULoad=0.00 ' + 'AvailableFeatures=f1,f2 ActiveFeatures=f1,f2 ' + 'Gres=gpu_mem:16280,gpu:1 NodeAddr=nid00001 ' + 'NodeHostName=nid00001 Version=10.00 OS=Linux ' + 'RealMemory=32220 AllocMem=0 FreeMem=10000 ' + 'Sockets=1 Boards=1 State=IDLE ' + 'ThreadsPerCore=2 TmpDisk=0 Weight=1 Owner=N/A ' + 'MCS_label=N/A Partitions=p1,p2 ' + 'BootTime=01 Jan 2018 ' + 'SlurmdStartTime=01 Jan 2018 ' + 'CfgTRES=cpu=24,mem=32220M ' + 'AllocTRES= CapWatts=n/a CurrentWatts=100 ' + 'LowestJoules=100000000 ConsumedJoules=0 ' + 'ExtSensorsJoules=n/s ExtSensorsWatts=0 ' + 'ExtSensorsTemp=n/s Reason=Foo/ ' + 'failed [reframe_user@01 Jan 2018]' + ) + + +@pytest.fixture +def slurm_node_drained(): + return _SlurmNode( + 'NodeName=nid00003 Arch=x86_64 CoresPerSocket=12 ' + 'CPUAlloc=0 CPUErr=0 CPUTot=24 CPULoad=0.00 ' + 'AvailableFeatures=f1,f2 ActiveFeatures=f1,f2 ' + 'Gres=gpu_mem:16280,gpu:1 NodeAddr=nid00001 ' + 'NodeHostName=nid00001 Version=10.00 OS=Linux ' + 'RealMemory=32220 AllocMem=0 FreeMem=10000 ' + 'Sockets=1 Boards=1 State=IDLE+DRAIN ' + 'ThreadsPerCore=2 TmpDisk=0 Weight=1 Owner=N/A ' + 'MCS_label=N/A Partitions=p1,p2 ' + 'BootTime=01 Jan 2018 ' + 'SlurmdStartTime=01 Jan 2018 ' + 'CfgTRES=cpu=24,mem=32220M ' + 'AllocTRES= CapWatts=n/a CurrentWatts=100 ' + 'LowestJoules=100000000 ConsumedJoules=0 ' + 'ExtSensorsJoules=n/s ExtSensorsWatts=0 ' + 'ExtSensorsTemp=n/s Reason=Foo/ ' + 'failed [reframe_user@01 Jan 2018]' + ) + + +@pytest.fixture +def slurm_node_nopart(): + return _SlurmNode( + 'NodeName=nid00004 Arch=x86_64 CoresPerSocket=12 ' + 'CPUAlloc=0 CPUErr=0 CPUTot=24 CPULoad=0.00 ' + 'AvailableFeatures=f1,f2 ActiveFeatures=f1,f2 ' + 'Gres=gpu_mem:16280,gpu:1 NodeAddr=nid00001 ' + 'NodeHostName=nid00001 Version=10.00 OS=Linux ' + 'RealMemory=32220 AllocMem=0 FreeMem=10000 ' + 'Sockets=1 Boards=1 State=IDLE+DRAIN ' + 'ThreadsPerCore=2 TmpDisk=0 Weight=1 Owner=N/A ' + 'MCS_label=N/A BootTime=01 Jan 2018 ' + 'SlurmdStartTime=01 Jan 2018 ' + 'CfgTRES=cpu=24,mem=32220M ' + 'AllocTRES= CapWatts=n/a CurrentWatts=100 ' + 'LowestJoules=100000000 ConsumedJoules=0 ' + 'ExtSensorsJoules=n/s ExtSensorsWatts=0 ' + 'ExtSensorsTemp=n/s Reason=Foo/ ' + 'failed [reframe_user@01 Jan 2018]' + ) + + +def test_slurm_node_noname(): + with pytest.raises(JobError): + _SlurmNode( 'Arch=x86_64 CoresPerSocket=12 ' 'CPUAlloc=0 CPUErr=0 CPUTot=24 CPULoad=0.00 ' 'AvailableFeatures=f1,f2 ActiveFeatures=f1,f2 ' @@ -995,46 +944,72 @@ def setUp(self): 'failed [reframe_user@01 Jan 2018]' ) + +def test_slurm_node_states(slurm_node_allocated, + slurm_node_idle, + slurm_node_drained): + assert slurm_node_allocated.states == {'ALLOCATED'} + assert slurm_node_idle.states == {'IDLE'} + assert slurm_node_drained.states == {'IDLE', 'DRAIN'} + + +def test_slurm_node_equals(slurm_node_allocated, slurm_node_idle): + assert slurm_node_allocated == _SlurmNode(slurm_node_allocated.descr) + assert slurm_node_allocated != slurm_node_idle + + +def test_slurm_node_attributes(slurm_node_allocated, slurm_node_nopart): + assert slurm_node_allocated.name == 'nid00001' + assert slurm_node_allocated.partitions == {'p1', 'p2'} + assert slurm_node_allocated.active_features == {'f1', 'f2'} + assert slurm_node_nopart.name == 'nid00004' + assert slurm_node_nopart.partitions == set() + assert slurm_node_nopart.active_features == {'f1', 'f2'} + + +def test_hash(slurm_node_allocated): + assert (hash(slurm_node_allocated) == + hash(_SlurmNode(slurm_node_allocated.descr))) + + +def test_str(slurm_node_allocated): + assert 'nid00001' == str(slurm_node_allocated) + + +def test_slurm_node_is_available(slurm_node_allocated, + slurm_node_idle, + slurm_node_drained, + slurm_node_nopart): + assert not slurm_node_allocated.is_available() + assert slurm_node_idle.is_available() + assert not slurm_node_drained.is_available() + assert not slurm_node_nopart.is_available() + + +def test_slurm_node_is_down(slurm_node_allocated, + slurm_node_idle, + slurm_node_nopart): + assert not slurm_node_allocated.is_down() + assert not slurm_node_idle.is_down() + assert slurm_node_nopart.is_down() + + +class TestSlurmNode: + def setUp(self): + idle_node_description = ( + ) + + idle_drained_node_description = ( + ) + + no_partition_node_description = ( + ) + + self.no_name_node_description = ( + ) + self.allocated_node = _SlurmNode(allocated_node_description) self.allocated_node_copy = _SlurmNode(allocated_node_description) self.idle_node = _SlurmNode(idle_node_description) self.idle_drained = _SlurmNode(idle_drained_node_description) self.no_partition_node = _SlurmNode(no_partition_node_description) - - def test_no_node_name(self): - with pytest.raises(JobError): - _SlurmNode(self.no_name_node_description) - - def test_states(self): - assert self.allocated_node.states == {'ALLOCATED'} - assert self.idle_node.states == {'IDLE'} - assert self.idle_drained.states == {'IDLE', 'DRAIN'} - - def test_equals(self): - assert self.allocated_node == self.allocated_node_copy - assert self.allocated_node != self.idle_node - - def test_hash(self): - assert hash(self.allocated_node) == hash(self.allocated_node_copy) - - def test_attributes(self): - assert self.allocated_node.name == 'nid00001' - assert self.allocated_node.partitions == {'p1', 'p2'} - assert self.allocated_node.active_features == {'f1', 'f2'} - assert self.no_partition_node.name == 'nid00004' - assert self.no_partition_node.partitions == set() - assert self.no_partition_node.active_features == {'f1', 'f2'} - - def test_str(self): - assert 'nid00001' == str(self.allocated_node) - - def test_is_available(self): - assert not self.allocated_node.is_available() - assert self.idle_node.is_available() - assert not self.idle_drained.is_available() - assert not self.no_partition_node.is_available() - - def test_is_down(self): - assert not self.allocated_node.is_down() - assert not self.idle_node.is_down() - assert self.no_partition_node.is_down()