From d70aeda5b4e39110c1159eded891a84580b35898 Mon Sep 17 00:00:00 2001 From: Matthew Thompson Date: Mon, 11 Sep 2023 14:37:08 -0400 Subject: [PATCH 1/7] Updates for Milan Support at NCCS --- CHANGELOG.md | 2 ++ build.csh | 9 ++++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f934664..1f77e75 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Removed ### Added +- Added support for Milan at NCCS + ## [4.19.0] - 2023-07-27 ### Changed diff --git a/build.csh b/build.csh index 2e63f5b..491b483 100755 --- a/build.csh +++ b/build.csh @@ -128,6 +128,7 @@ while ($#argv) # specify node type #------------------ + if ("$1" == "-mil") set nodeTYPE = "Milan" if ("$1" == "-rom") set nodeTYPE = "Rome" if ("$1" == "-cas") set nodeTYPE = "CascadeLake" if ("$1" == "-sky") set nodeTYPE = "Skylake" @@ -297,10 +298,12 @@ if ($SITE == NCCS) then if ($nT == any) @ NCPUS_DFLT = 40 if ($nT == sky) @ NCPUS_DFLT = 40 if ($nT == cas) @ NCPUS_DFLT = 48 + if ($nT == mil) @ NCPUS_DFLT = 128 if ($nT == any) set proc = 'any' if ($nT == sky) set proc = 'sky' if ($nT == cas) set proc = 'cas' + if ($nT == mil) set proc = 'mil' # If we are using GNU at NCCS, we can*only* use the cas queue # as OpenMPI is only built for Infiniband @@ -318,7 +321,10 @@ if ($SITE == NCCS) then set queue = '--qos=debug' endif - if ("$partition" == "") then + # Milan's must be submitted to the scutest partition + if ($nT == mil) then + set partition = '--partition=scutest' + else if ("$partition" == "") then set partition = '--partition=compute' endif @@ -891,6 +897,7 @@ flagged options -account account send batch job to account -walltime hh:mm:ss time to use as batch walltime at job submittal + -mil compile on Milan nodes (only at NCCS) -rom compile on Rome nodes (only at NAS) -cas compile on Cascade Lake nodes -sky compile on Skylake nodes (default at NAS) From 9bc19dfe352f266f0b245a608e4de9d90d8d2202 Mon Sep 17 00:00:00 2001 From: Matthew Thompson Date: Tue, 12 Sep 2023 09:07:27 -0400 Subject: [PATCH 2/7] Commit g5_modules for SLES15 --- g5_modules | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/g5_modules b/g5_modules index 4be7bd2..7006e55 100755 --- a/g5_modules +++ b/g5_modules @@ -41,7 +41,7 @@ # 21Jul2008 Takacs New modules and BASEDIR on discover after OS upgrade # 13Apr2009 Stassi Updated for pleiades # 22Apr2010 Kokron Updated for Fortuna-2.1 on pleiades -# 21Jul2011 Kokron Overlay older MKL module as on discover to gain reproducible results from dgeev in GSI +# 21Jul2011 Kokron Overlay older MKL module as on discover to gain reproducible results from dgeev in GSI # 24Aug2012 Stassi Added sh option to write bash source-able file # 03Nov2016 Thompson Remove JIBB ######################################################################## @@ -123,20 +123,20 @@ if ( $site == NCCS ) then set mod1 = GEOSenv - set mod2 = comp/gcc/11.2.0 + set mod2 = comp/gcc/12.3.0 set mod3 = comp/intel/2021.6.0 set mod4 = mpi/impi/2021.6.0 - set mod5 = python/GEOSpyD/Min4.11.0_py3.9_AND_Min4.8.3_py2.7 + set mod5 = python/GEOSpyD/Min23.5.2-0_py3.11 - set basedir = /discover/swdev/gmao_SIteam/Baselibs/ESMA-Baselibs-7.14.0/x86_64-pc-linux-gnu/ifort_2021.6.0-intelmpi_2021.6.0 + set basedir = /discover/swdev/gmao_SIteam/Baselibs/ESMA-Baselibs-7.14.0/x86_64-pc-linux-gnu/ifort_2021.6.0-intelmpi_2021.6.0-SLES15 set mods = ( $mod1 $mod2 $mod3 $mod4 $mod5 ) set modinit = /usr/share/modules/init/csh set loadmodules = 0 - set usemod1 = /discover/swdev/gmao_SIteam/modulefiles-SLES12 + set usemod1 = /discover/swdev/gmao_SIteam/modulefiles-SLES15 set usemods = ( $usemod1 ) set usemodules = 1 @@ -389,7 +389,7 @@ DESCRIPTION add the BASEDIR lib directory to LD_LIBRARY_PATH (if necessary), and will load library modules when sourced. - If the script is called with "basedir", "modules", "modinit", or + If the script is called with "basedir", "modules", "modinit", or "loadmodules", then it will echo the values to standard output without modifying the environment. From 5b25ec15e90b9fafe065ccca260cba65f9698133 Mon Sep 17 00:00:00 2001 From: Matthew Thompson Date: Wed, 13 Sep 2023 12:58:50 -0400 Subject: [PATCH 3/7] Move to Intel MPI 2021.10 --- g5_modules | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/g5_modules b/g5_modules index 7006e55..55fee72 100755 --- a/g5_modules +++ b/g5_modules @@ -126,11 +126,11 @@ if ( $site == NCCS ) then set mod2 = comp/gcc/12.3.0 set mod3 = comp/intel/2021.6.0 - set mod4 = mpi/impi/2021.6.0 + set mod4 = mpi/impi/2021.10.0 set mod5 = python/GEOSpyD/Min23.5.2-0_py3.11 - set basedir = /discover/swdev/gmao_SIteam/Baselibs/ESMA-Baselibs-7.14.0/x86_64-pc-linux-gnu/ifort_2021.6.0-intelmpi_2021.6.0-SLES15 + set basedir = /discover/swdev/gmao_SIteam/Baselibs/ESMA-Baselibs-7.14.0/x86_64-pc-linux-gnu/ifort_2021.6.0-intelmpi_2021.10.0-SLES15 set mods = ( $mod1 $mod2 $mod3 $mod4 $mod5 ) set modinit = /usr/share/modules/init/csh From 21ef5b563d445a31cc0a6c96c7783aa8c07a9edd Mon Sep 17 00:00:00 2001 From: Matthew Thompson Date: Wed, 13 Sep 2023 14:27:15 -0400 Subject: [PATCH 4/7] Move to use OpenMPI by default --- g5_modules | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/g5_modules b/g5_modules index 55fee72..6a7a281 100755 --- a/g5_modules +++ b/g5_modules @@ -41,7 +41,7 @@ # 21Jul2008 Takacs New modules and BASEDIR on discover after OS upgrade # 13Apr2009 Stassi Updated for pleiades # 22Apr2010 Kokron Updated for Fortuna-2.1 on pleiades -# 21Jul2011 Kokron Overlay older MKL module as on discover to gain reproducible results from dgeev in GSI +# 21Jul2011 Kokron Overlay older MKL module as on discover to gain reproducible results from dgeev in GSI # 24Aug2012 Stassi Added sh option to write bash source-able file # 03Nov2016 Thompson Remove JIBB ######################################################################## @@ -126,11 +126,11 @@ if ( $site == NCCS ) then set mod2 = comp/gcc/12.3.0 set mod3 = comp/intel/2021.6.0 - set mod4 = mpi/impi/2021.10.0 + set mod4 = mpi/openmpi/4.1.5/intel-2021.6.0 set mod5 = python/GEOSpyD/Min23.5.2-0_py3.11 - set basedir = /discover/swdev/gmao_SIteam/Baselibs/ESMA-Baselibs-7.14.0/x86_64-pc-linux-gnu/ifort_2021.6.0-intelmpi_2021.10.0-SLES15 + set basedir = /discover/swdev/gmao_SIteam/Baselibs/ESMA-Baselibs-7.14.0/x86_64-pc-linux-gnu/ifort_2021.6.0-openmpi_4.1.5-SLES15 set mods = ( $mod1 $mod2 $mod3 $mod4 $mod5 ) set modinit = /usr/share/modules/init/csh @@ -389,7 +389,7 @@ DESCRIPTION add the BASEDIR lib directory to LD_LIBRARY_PATH (if necessary), and will load library modules when sourced. - If the script is called with "basedir", "modules", "modinit", or + If the script is called with "basedir", "modules", "modinit", or "loadmodules", then it will echo the values to standard output without modifying the environment. From 3fac633a5c08d22020b05115917612e4ddf68f91 Mon Sep 17 00:00:00 2001 From: Matthew Thompson Date: Thu, 12 Oct 2023 13:07:23 -0400 Subject: [PATCH 5/7] Update changelog --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1f77e75..3eee6de 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Removed ### Added +## [4.20.0] - 2023-10-14 + +### Added + - Added support for Milan at NCCS + - Uses Open MPI 4.1.5 on SCU17 rather than Intel MPI due to issues with Intel MPI ## [4.19.0] - 2023-07-27 From 4e67e5119603bffa10a288167015764cb923d09b Mon Sep 17 00:00:00 2001 From: Matthew Thompson Date: Thu, 12 Oct 2023 13:37:07 -0400 Subject: [PATCH 6/7] Use combo g5_modules --- g5_modules | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/g5_modules b/g5_modules index 6a7a281..b529ab7 100755 --- a/g5_modules +++ b/g5_modules @@ -74,6 +74,9 @@ if (($node =~ discover*) || ($node =~ borg*) || \ set site = "NCCS" + # NCCS now has both SLES15 and SLES12 machines + set OS_VERSION=`grep VERSION_ID /etc/os-release | cut -d= -f2 | cut -d. -f1 | sed 's/"//g'` + else if (($node =~ pfe*) || ($node =~ tfe*) || \ ($node =~ r[0-9]*i[0-9]*n[0-9]*) || \ ($node =~ r[0-9]*c[0-9]*t[0-9]*n[0-9]*)) then @@ -123,20 +126,30 @@ if ( $site == NCCS ) then set mod1 = GEOSenv - set mod2 = comp/gcc/12.3.0 - set mod3 = comp/intel/2021.6.0 + if ( $OS_VERSION == 12 ) then + + set mod2 = comp/gcc/11.2.0 + set mod3 = comp/intel/2021.6.0 + set mod4 = mpi/impi/2021.6.0 + set mod5 = python/GEOSpyD/Min4.11.0_py3.9_AND_Min4.8.3_py2.7 + set basedir = /discover/swdev/gmao_SIteam/Baselibs/ESMA-Baselibs-7.14.0/x86_64-pc-linux-gnu/ifort_2021.6.0-intelmpi_2021.6.0 + set usemod1 = /discover/swdev/gmao_SIteam/modulefiles-SLES12 - set mod4 = mpi/openmpi/4.1.5/intel-2021.6.0 + else - set mod5 = python/GEOSpyD/Min23.5.2-0_py3.11 + set mod2 = comp/gcc/12.3.0 + set mod3 = comp/intel/2021.6.0 + set mod4 = mpi/openmpi/4.1.5/intel-2021.6.0 + set mod5 = python/GEOSpyD/Min23.5.2-0_py3.11 + set basedir = /discover/swdev/gmao_SIteam/Baselibs/ESMA-Baselibs-7.14.0/x86_64-pc-linux-gnu/ifort_2021.6.0-openmpi_4.1.5-SLES15 + set usemod1 = /discover/swdev/gmao_SIteam/modulefiles-SLES15 - set basedir = /discover/swdev/gmao_SIteam/Baselibs/ESMA-Baselibs-7.14.0/x86_64-pc-linux-gnu/ifort_2021.6.0-openmpi_4.1.5-SLES15 + endif set mods = ( $mod1 $mod2 $mod3 $mod4 $mod5 ) set modinit = /usr/share/modules/init/csh set loadmodules = 0 - set usemod1 = /discover/swdev/gmao_SIteam/modulefiles-SLES15 set usemods = ( $usemod1 ) set usemodules = 1 From 892d51fbaeb8d407360800e9c6f0b773c2036fd3 Mon Sep 17 00:00:00 2001 From: Matthew Thompson Date: Thu, 12 Oct 2023 14:00:16 -0400 Subject: [PATCH 7/7] Remove reference to scutest --- build.csh | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/build.csh b/build.csh index 491b483..68545ea 100755 --- a/build.csh +++ b/build.csh @@ -321,10 +321,7 @@ if ($SITE == NCCS) then set queue = '--qos=debug' endif - # Milan's must be submitted to the scutest partition - if ($nT == mil) then - set partition = '--partition=scutest' - else if ("$partition" == "") then + if ("$partition" == "") then set partition = '--partition=compute' endif