From 8ffee79d52c927fc1cfff787bd187a38df5a664a Mon Sep 17 00:00:00 2001 From: rafael Date: Tue, 19 Feb 2019 11:24:40 +0100 Subject: [PATCH 01/11] add fulen to config --- config/cscs.py | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/config/cscs.py b/config/cscs.py index 896769b834..cccbeaeee5 100644 --- a/config/cscs.py +++ b/config/cscs.py @@ -99,6 +99,42 @@ class ReframeSettings: } }, + 'fulen': { + 'descr': 'Fulen', + 'hostnames': ['fulen'], + 'modules_system': 'tmod', + 'resourcesdir': '/apps/common/UES/reframe/resources', + 'partitions': { + 'login': { + 'scheduler': 'local', + 'environs': ['PrgEnv-gnu'], + 'descr': 'Login nodes', + 'max_jobs': 1 + }, + + 'normal': { + 'scheduler': 'nativeslurm', + 'environs': ['PrgEnv-gnu'], + 'descr': ('Compute nodes - ' + 'default partition'), + }, + + 'fat': { + 'scheduler': 'nativeslurm', + 'environs': ['PrgEnv-gnu'], + 'access': ['--partition fat'], + 'descr': ('High-memory compute nodes'), + }, + + 'gpu': { + 'scheduler': 'nativeslurm', + 'environs': ['PrgEnv-gnu'], + 'access': ['--partition gpu'], + 'descr': ('Hybrid compute nodes'), + }, + } + }, + 'kesch': { 'descr': 'Kesch MCH', 'hostnames': ['keschln-\d+'], From 494d0156d8f6172ef2b605db3baf5388ab631371 Mon Sep 17 00:00:00 2001 From: rafael Date: Tue, 19 Feb 2019 11:41:34 +0100 Subject: [PATCH 02/11] add fulen to Jenkinsfile --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index d1a1557ee4..ebf64f675b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -3,7 +3,7 @@ def dirPrefix = 'reframe-ci' def loginBash = '#!/bin/bash -l' def bashScript = 'ci-scripts/ci-runner.bash' -def machinesList = ['daint', 'dom', 'kesch'] +def machinesList = ['daint', 'dom', 'kesch', 'fulen'] def machinesToRun = machinesList def runTests = true def uniqueID From ad53763afb4075c92f94592bf223c7a6e12f9dad Mon Sep 17 00:00:00 2001 From: Theofilos Manitaras Date: Tue, 19 Feb 2019 11:51:21 +0100 Subject: [PATCH 03/11] Export module function for fulen in Jenkinsfile --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index ebf64f675b..459e9462f1 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -93,7 +93,7 @@ stage('Unittest') { echo \$SCRATCH""").trim() def reframeDir = "${scratch}/${dirPrefix}-${machineName}-${uniqueID}" def moduleDefinition = '' - if (machineName == 'leone') { + if (machineName in ['leone', 'fulen']) { moduleDefinition = '''module() { eval `/usr/bin/modulecmd bash $*`; } export -f module''' } From 4a5ddf4211c95f9733b99664501f96b42c000716 Mon Sep 17 00:00:00 2001 From: Theofilos Manitaras Date: Tue, 19 Feb 2019 14:21:27 +0100 Subject: [PATCH 04/11] Disable fulen from ci --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 459e9462f1..bed427262c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -3,7 +3,7 @@ def dirPrefix = 'reframe-ci' def loginBash = '#!/bin/bash -l' def bashScript = 'ci-scripts/ci-runner.bash' -def machinesList = ['daint', 'dom', 'kesch', 'fulen'] +def machinesList = ['daint', 'dom', 'kesch'] def machinesToRun = machinesList def runTests = true def uniqueID From edde301fac75fdeb3bd819ed2f3399fd5416df97 Mon Sep 17 00:00:00 2001 From: Theofilos Manitaras Date: Fri, 22 Feb 2019 09:41:32 +0100 Subject: [PATCH 05/11] Run generic unittests on fulen --- Jenkinsfile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index bed427262c..a485575d6a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -3,7 +3,7 @@ def dirPrefix = 'reframe-ci' def loginBash = '#!/bin/bash -l' def bashScript = 'ci-scripts/ci-runner.bash' -def machinesList = ['daint', 'dom', 'kesch'] +def machinesList = ['daint', 'dom', 'kesch', 'fulen'] def machinesToRun = machinesList def runTests = true def uniqueID @@ -97,11 +97,13 @@ stage('Unittest') { moduleDefinition = '''module() { eval `/usr/bin/modulecmd bash $*`; } export -f module''' } + # Run only the generic unit tests on fulen + def genericTestFlag = machineName in ['fulen'] ? '-g' : '' dir(reframeDir) { checkout scm sh("""${loginBash} ${moduleDefinition} - bash ${reframeDir}/${bashScript} -f ${reframeDir} -i ''""") + bash ${reframeDir}/${bashScript} -f ${reframeDir} -i '' ${genericTestFlag}""") } } } From 3695ff2db951b6bd2c0b19c172d40b3a9b65b0f3 Mon Sep 17 00:00:00 2001 From: Theofilos Manitaras Date: Fri, 22 Feb 2019 09:43:12 +0100 Subject: [PATCH 06/11] Fix bug in comment --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index a485575d6a..f5f7dc92a8 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -97,7 +97,7 @@ stage('Unittest') { moduleDefinition = '''module() { eval `/usr/bin/modulecmd bash $*`; } export -f module''' } - # Run only the generic unit tests on fulen + // Run only the generic unit tests on fulen def genericTestFlag = machineName in ['fulen'] ? '-g' : '' dir(reframeDir) { checkout scm From d671ae4a02d968016952139e2704dd5f6f93cef2 Mon Sep 17 00:00:00 2001 From: Theofilos Manitaras Date: Fri, 22 Feb 2019 10:16:25 +0100 Subject: [PATCH 07/11] Add option for local-only tests in ci script --- Jenkinsfile | 4 ++-- ci-scripts/ci-runner.bash | 15 ++++++++++++--- 2 files changed, 14 insertions(+), 5 deletions(-) mode change 100644 => 100755 ci-scripts/ci-runner.bash diff --git a/Jenkinsfile b/Jenkinsfile index f5f7dc92a8..024ee18f55 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -98,12 +98,12 @@ stage('Unittest') { export -f module''' } // Run only the generic unit tests on fulen - def genericTestFlag = machineName in ['fulen'] ? '-g' : '' + def localOnlyFlag = machineName in ['fulen'] ? '-L' : '' dir(reframeDir) { checkout scm sh("""${loginBash} ${moduleDefinition} - bash ${reframeDir}/${bashScript} -f ${reframeDir} -i '' ${genericTestFlag}""") + bash ${reframeDir}/${bashScript} -f ${reframeDir} -i '' ${localOnlyFlag}""") } } } diff --git a/ci-scripts/ci-runner.bash b/ci-scripts/ci-runner.bash old mode 100644 new mode 100755 index 76cfc415b5..145fcd9309 --- a/ci-scripts/ci-runner.bash +++ b/ci-scripts/ci-runner.bash @@ -5,6 +5,7 @@ CI_FOLDER="" CI_GENERIC=0 CI_TUTORIAL=0 CI_EXITCODE=0 +CI_LOCAL=0 TERM="${TERM:-xterm}" PROFILE="" MODULEUSE="" @@ -26,6 +27,7 @@ Usage: $(tput setaf 1)$scriptname$(tput sgr0) $(tput setaf 3)[OPTIONS]$(tput sgr $(tput setaf 3)-m | --module-use$(tput sgr0) $(tput setaf 1)ARGS$(tput sgr0) executes module use of the give folder before loading the regression $(tput setaf 3)-g | --generic-only$(tput sgr0) executes unit tests using the generic configuration $(tput setaf 3)-t | --tutorial-only$(tput sgr0) executes only the modified/new tutorial tests + $(tput setaf 3)-L | --local-only$(tput sgr0) executes the unit tests without a configuration $(tput setaf 3)-h | --help$(tput sgr0) prints this help and exits EOF @@ -66,8 +68,8 @@ run_serial_user_checks() ### Main script ### -shortopts="h,g,t,f:,i:,l:,m:" -longopts="help,generic-only,tutorial-only,folder:,invocation:,load-profile:,module-use:" +shortopts="h,g,t,L,f:,i:,l:,m:" +longopts="help,generic-only,tutorial-only,local-only,folder:,invocation:,load-profile:,module-use:" eval set -- $(getopt -o ${shortopts} -l ${longopts} \ -n ${scriptname} -- "$@" 2> /dev/null) @@ -102,6 +104,9 @@ while [ $# -ne 0 ]; do -t | --tutorial-only) shift CI_TUTORIAL=1 ;; + -L | --local-only) + shift + CI_LOCAL=1 ;; --) ;; *) @@ -180,7 +185,11 @@ else echo "Running unit tests" echo "==================" - checked_exec ./test_reframe.py --rfm-user-config=config/cscs.py + if [ $CI_LOCAL -eq 1 ]; then + checked_exec ./test_reframe.py + else + checked_exec ./test_reframe.py --rfm-user-config=config/cscs.py + fi echo "===================================" echo "Running unit tests with PBS backend" From 24e27b88f675dddb5a6dcd24ee9a74115e545c6f Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Sat, 9 Mar 2019 14:27:51 +0100 Subject: [PATCH 08/11] Revert changes in CI-related stuff for Fulen --- Jenkinsfile | 8 +++----- ci-scripts/ci-runner.bash | 15 +++------------ 2 files changed, 6 insertions(+), 17 deletions(-) mode change 100755 => 100644 ci-scripts/ci-runner.bash diff --git a/Jenkinsfile b/Jenkinsfile index 024ee18f55..5cf5c6ff19 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -3,7 +3,7 @@ def dirPrefix = 'reframe-ci' def loginBash = '#!/bin/bash -l' def bashScript = 'ci-scripts/ci-runner.bash' -def machinesList = ['daint', 'dom', 'kesch', 'fulen'] +def machinesList = ['daint', 'kesch'] def machinesToRun = machinesList def runTests = true def uniqueID @@ -93,17 +93,15 @@ stage('Unittest') { echo \$SCRATCH""").trim() def reframeDir = "${scratch}/${dirPrefix}-${machineName}-${uniqueID}" def moduleDefinition = '' - if (machineName in ['leone', 'fulen']) { + if (machineName == 'leone') { moduleDefinition = '''module() { eval `/usr/bin/modulecmd bash $*`; } export -f module''' } - // Run only the generic unit tests on fulen - def localOnlyFlag = machineName in ['fulen'] ? '-L' : '' dir(reframeDir) { checkout scm sh("""${loginBash} ${moduleDefinition} - bash ${reframeDir}/${bashScript} -f ${reframeDir} -i '' ${localOnlyFlag}""") + bash ${reframeDir}/${bashScript} -f ${reframeDir} -i ''""") } } } diff --git a/ci-scripts/ci-runner.bash b/ci-scripts/ci-runner.bash old mode 100755 new mode 100644 index 145fcd9309..76cfc415b5 --- a/ci-scripts/ci-runner.bash +++ b/ci-scripts/ci-runner.bash @@ -5,7 +5,6 @@ CI_FOLDER="" CI_GENERIC=0 CI_TUTORIAL=0 CI_EXITCODE=0 -CI_LOCAL=0 TERM="${TERM:-xterm}" PROFILE="" MODULEUSE="" @@ -27,7 +26,6 @@ Usage: $(tput setaf 1)$scriptname$(tput sgr0) $(tput setaf 3)[OPTIONS]$(tput sgr $(tput setaf 3)-m | --module-use$(tput sgr0) $(tput setaf 1)ARGS$(tput sgr0) executes module use of the give folder before loading the regression $(tput setaf 3)-g | --generic-only$(tput sgr0) executes unit tests using the generic configuration $(tput setaf 3)-t | --tutorial-only$(tput sgr0) executes only the modified/new tutorial tests - $(tput setaf 3)-L | --local-only$(tput sgr0) executes the unit tests without a configuration $(tput setaf 3)-h | --help$(tput sgr0) prints this help and exits EOF @@ -68,8 +66,8 @@ run_serial_user_checks() ### Main script ### -shortopts="h,g,t,L,f:,i:,l:,m:" -longopts="help,generic-only,tutorial-only,local-only,folder:,invocation:,load-profile:,module-use:" +shortopts="h,g,t,f:,i:,l:,m:" +longopts="help,generic-only,tutorial-only,folder:,invocation:,load-profile:,module-use:" eval set -- $(getopt -o ${shortopts} -l ${longopts} \ -n ${scriptname} -- "$@" 2> /dev/null) @@ -104,9 +102,6 @@ while [ $# -ne 0 ]; do -t | --tutorial-only) shift CI_TUTORIAL=1 ;; - -L | --local-only) - shift - CI_LOCAL=1 ;; --) ;; *) @@ -185,11 +180,7 @@ else echo "Running unit tests" echo "==================" - if [ $CI_LOCAL -eq 1 ]; then - checked_exec ./test_reframe.py - else - checked_exec ./test_reframe.py --rfm-user-config=config/cscs.py - fi + checked_exec ./test_reframe.py --rfm-user-config=config/cscs.py echo "===================================" echo "Running unit tests with PBS backend" From 15b780de4b3b5ef77014ae8475acefe48cbaa157 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Sat, 9 Mar 2019 14:36:55 +0100 Subject: [PATCH 09/11] Address PR comments --- config/cscs.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/config/cscs.py b/config/cscs.py index cccbeaeee5..c45215c9b0 100644 --- a/config/cscs.py +++ b/config/cscs.py @@ -101,7 +101,7 @@ class ReframeSettings: 'fulen': { 'descr': 'Fulen', - 'hostnames': ['fulen'], + 'hostnames': ['fulen-ln\d+'], 'modules_system': 'tmod', 'resourcesdir': '/apps/common/UES/reframe/resources', 'partitions': { @@ -115,22 +115,21 @@ class ReframeSettings: 'normal': { 'scheduler': 'nativeslurm', 'environs': ['PrgEnv-gnu'], - 'descr': ('Compute nodes - ' - 'default partition'), + 'descr': 'Compute nodes - default partition', }, 'fat': { 'scheduler': 'nativeslurm', 'environs': ['PrgEnv-gnu'], 'access': ['--partition fat'], - 'descr': ('High-memory compute nodes'), + 'descr': 'High-memory compute nodes', }, 'gpu': { 'scheduler': 'nativeslurm', 'environs': ['PrgEnv-gnu'], 'access': ['--partition gpu'], - 'descr': ('Hybrid compute nodes'), + 'descr': 'Hybrid compute nodes', }, } }, From c45eb4ded2875c663584240bd99ae1fd51c291ff Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Sat, 9 Mar 2019 14:38:34 +0100 Subject: [PATCH 10/11] Fix PEP8 issue --- config/cscs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/cscs.py b/config/cscs.py index c45215c9b0..317767287e 100644 --- a/config/cscs.py +++ b/config/cscs.py @@ -101,7 +101,7 @@ class ReframeSettings: 'fulen': { 'descr': 'Fulen', - 'hostnames': ['fulen-ln\d+'], + 'hostnames': [r'fulen-ln\d+'], 'modules_system': 'tmod', 'resourcesdir': '/apps/common/UES/reframe/resources', 'partitions': { From ac02031018b68ebf694e0f07eacd35ccb6207bdd Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Sat, 9 Mar 2019 14:42:55 +0100 Subject: [PATCH 11/11] Remove redundant spaces --- config/cscs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/cscs.py b/config/cscs.py index 317767287e..842a44445d 100644 --- a/config/cscs.py +++ b/config/cscs.py @@ -121,14 +121,14 @@ class ReframeSettings: 'fat': { 'scheduler': 'nativeslurm', 'environs': ['PrgEnv-gnu'], - 'access': ['--partition fat'], + 'access': ['--partition fat'], 'descr': 'High-memory compute nodes', }, 'gpu': { 'scheduler': 'nativeslurm', 'environs': ['PrgEnv-gnu'], - 'access': ['--partition gpu'], + 'access': ['--partition gpu'], 'descr': 'Hybrid compute nodes', }, }