From 1a04bf1d715788e734d943ca29c3602ed853be34 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 28 Feb 2025 06:44:57 -0700 Subject: [PATCH 1/5] Update site config for Derecho: use oneapi@2024.2.1 instead of intel@2021, gcc@12.2 instead of gcc@11.2; update cray-mpich --- configs/sites/tier1/derecho/compilers.yaml | 38 +++++-------------- configs/sites/tier1/derecho/packages.yaml | 5 --- configs/sites/tier1/derecho/packages_gcc.yaml | 6 +-- .../sites/tier1/derecho/packages_intel.yaml | 23 ----------- .../sites/tier1/derecho/packages_oneapi.yaml | 25 ++++++++++++ 5 files changed, 37 insertions(+), 60 deletions(-) delete mode 100644 configs/sites/tier1/derecho/packages_intel.yaml create mode 100644 configs/sites/tier1/derecho/packages_oneapi.yaml diff --git a/configs/sites/tier1/derecho/compilers.yaml b/configs/sites/tier1/derecho/compilers.yaml index db05f7f08..bca611593 100644 --- a/configs/sites/tier1/derecho/compilers.yaml +++ b/configs/sites/tier1/derecho/compilers.yaml @@ -1,22 +1,22 @@ compilers:: - compiler: - spec: intel@2021.10.0 + spec: oneapi@2024.2.1 paths: - cc: /glade/u/apps/common/23.08/spack/opt/spack/intel-oneapi-compilers/2023.2.1/compiler/2023.2.1/linux/bin/intel64/icc - cxx: /glade/u/apps/common/23.08/spack/opt/spack/intel-oneapi-compilers/2023.2.1/compiler/2023.2.1/linux/bin/intel64/icpc - f77: /glade/u/apps/common/23.08/spack/opt/spack/intel-oneapi-compilers/2023.2.1/compiler/2023.2.1/linux/bin/intel64/ifort - fc: /glade/u/apps/common/23.08/spack/opt/spack/intel-oneapi-compilers/2023.2.1/compiler/2023.2.1/linux/bin/intel64/ifort + cc: /glade/u/apps/common/23.08/spack/opt/spack/intel-oneapi-compilers/2024.2.1/compiler/2024.2/bin/icx + cxx: /glade/u/apps/common/23.08/spack/opt/spack/intel-oneapi-compilers/2024.2.1/compiler/2024.2/bin/icpx + f77: /glade/u/apps/common/23.08/spack/opt/spack/intel-oneapi-compilers/2024.2.1/compiler/2024.2/bin/ifort + fc: /glade/u/apps/common/23.08/spack/opt/spack/intel-oneapi-compilers/2024.2.1/compiler/2024.2/bin/ifort flags: {} operating_system: sles15 target: x86_64 modules: - ncarenv/23.09 - - intel-classic/2023.2.1 + - intel-oneapi/2024.2.1 environment: prepend_path: - PATH: '/opt/cray/pe/gcc/11.2.0/snos/bin' - CPATH: '/opt/cray/pe/gcc/11.2.0/snos/include' - LD_LIBRARY_PATH: '/glade/u/apps/common/23.08/spack/opt/spack/intel-oneapi-compilers/2023.2.1/compiler/2023.2.1/linux/compiler/lib/intel64_lin:/opt/cray/pe/gcc/11.2.0/snos/lib:/opt/cray/pe/gcc/11.2.0/snos/lib64' + PATH: '/opt/cray/pe/gcc/12.2.0/snos/bin' + CPATH: '/opt/cray/pe/gcc/12.2.0/snos/include' + LD_LIBRARY_PATH: '/opt/cray/pe/gcc/12.2.0/snos/lib:/opt/cray/pe/gcc/12.2.0/snos/lib64' set: # https://github.com/JCSDA/spack-stack/issues/957 FI_CXI_RX_MATCH_MODE: 'hybrid' @@ -41,23 +41,3 @@ compilers:: # https://github.com/JCSDA/spack-stack/issues/957 FI_CXI_RX_MATCH_MODE: 'hybrid' extra_rpaths: [] -# Needed for intel -- compiler: - spec: gcc@11.2.0 - paths: - cc: /opt/cray/pe/gcc/11.2.0/bin/gcc - cxx: /opt/cray/pe/gcc/11.2.0/bin/g++ - f77: /opt/cray/pe/gcc/11.2.0/bin/gfortran - fc: /opt/cray/pe/gcc/11.2.0/bin/gfortran - flags: {} - operating_system: sles15 - target: x86_64 - modules: - - ncarenv/23.09 - - gcc/11.2.0 - environment: - set: - # https://github.com/JCSDA/spack-stack/issues/957 - FI_CXI_RX_MATCH_MODE: 'hybrid' - extra_rpaths: [] - diff --git a/configs/sites/tier1/derecho/packages.yaml b/configs/sites/tier1/derecho/packages.yaml index ad9e5f303..11f4d566f 100644 --- a/configs/sites/tier1/derecho/packages.yaml +++ b/configs/sites/tier1/derecho/packages.yaml @@ -7,11 +7,6 @@ packages: esmf: require: - 'esmf_os=Linux esmf_comm=mpich' - # py-scipy 1.12 does not build with intel on Derecho, pin to 1.11.4 - # use same version with gcc for consistency on the machine - py-scipy: - require: - - '@1.11.4' ### All other external packages listed alphabetically autoconf: diff --git a/configs/sites/tier1/derecho/packages_gcc.yaml b/configs/sites/tier1/derecho/packages_gcc.yaml index f84016fe8..0c05cbded 100644 --- a/configs/sites/tier1/derecho/packages_gcc.yaml +++ b/configs/sites/tier1/derecho/packages_gcc.yaml @@ -2,14 +2,14 @@ packages: all: compiler:: [gcc@12.2.0] providers: - mpi:: [cray-mpich@8.1.25] + mpi:: [cray-mpich@8.1.29] mpi: buildable: False cray-mpich: externals: - - spec: cray-mpich@8.1.25%gcc@12.2.0 +wrappers + - spec: cray-mpich@8.1.29%gcc@12.2.0 +wrappers modules: - craype/2.7.20 - - cray-mpich/8.1.25 + - cray-mpich/8.1.29 - libfabric/1.15.2.0 - cray-pals/1.2.11 diff --git a/configs/sites/tier1/derecho/packages_intel.yaml b/configs/sites/tier1/derecho/packages_intel.yaml deleted file mode 100644 index 2d870e136..000000000 --- a/configs/sites/tier1/derecho/packages_intel.yaml +++ /dev/null @@ -1,23 +0,0 @@ -packages: - all: - compiler:: [intel@2021.10.0,gcc@11.2.0] - providers: - mpi:: [cray-mpich@8.1.25] - mpi: - buildable: False - cray-mpich: - externals: - - spec: cray-mpich@8.1.25%intel@2021.10.0 +wrappers - modules: - - craype/2.7.20 - - cray-mpich/8.1.25 - - libfabric/1.15.2.0 - - cray-pals/1.2.11 - intel-oneapi-mkl: - # No intel-oneapi-mkl installed on Derecho, install via spack. - # Prefer the version that Intel releases with the above compiler - prefer: - - '@2023.2.0' - # Newer versions of patchelf (tested 0.18.0) don't build with Intel on Derecho - patchelf: - version:: ['0.17.2'] diff --git a/configs/sites/tier1/derecho/packages_oneapi.yaml b/configs/sites/tier1/derecho/packages_oneapi.yaml new file mode 100644 index 000000000..bfb807e6e --- /dev/null +++ b/configs/sites/tier1/derecho/packages_oneapi.yaml @@ -0,0 +1,25 @@ +packages: + all: + compiler:: [oneapi@2024.2.1, gcc@12.2.0] + providers: + mpi:: [cray-mpich@8.1.29] + mpi: + buildable: False + cray-mpich: + externals: + - spec: cray-mpich@8.1.29%oneapi@2024.2.1 +wrappers + modules: + - craype/2.7.20 + - cray-mpich/8.1.29 + - libfabric/1.15.2.0 + - cray-pals/1.2.11 + intel-oneapi-runtime: + buildable: False + externals: + - spec: intel-oneapi-runtime@2024.2.1%oneapi@2024.2.1 + prefix: /glade/u/apps/common/23.08/spack/opt/spack/intel-oneapi-compilers/2024.2.1 + intel-oneapi-mkl: + # No intel-oneapi-mkl installed on Derecho, install via spack. + # Prefer the version that Intel releases with the above compiler + prefer: + - '@2024.2' From bda4a728e09f48e6ca5068a352220547b8e44e62 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 28 Feb 2025 20:22:14 -0700 Subject: [PATCH 2/5] For Derecho with GNU, cray-mpich version is 8.1.27 and not 8.1.29 --- configs/sites/tier1/derecho/packages_gcc.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/configs/sites/tier1/derecho/packages_gcc.yaml b/configs/sites/tier1/derecho/packages_gcc.yaml index 0c05cbded..332c6117b 100644 --- a/configs/sites/tier1/derecho/packages_gcc.yaml +++ b/configs/sites/tier1/derecho/packages_gcc.yaml @@ -2,14 +2,14 @@ packages: all: compiler:: [gcc@12.2.0] providers: - mpi:: [cray-mpich@8.1.29] + mpi:: [cray-mpich@8.1.27] mpi: buildable: False cray-mpich: externals: - - spec: cray-mpich@8.1.29%gcc@12.2.0 +wrappers + - spec: cray-mpich@8.1.27%gcc@12.2.0 +wrappers modules: - craype/2.7.20 - - cray-mpich/8.1.29 + - cray-mpich/8.1.27 - libfabric/1.15.2.0 - cray-pals/1.2.11 From 5b097982be9192cef70ee981bf2e42f02e5d9561 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 3 Mar 2025 19:39:33 -0800 Subject: [PATCH 3/5] Update submodule pointer for spack (crtm 3.1.1 bug fixes) --- spack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spack b/spack index 5c04e4830..60244b842 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit 5c04e483088be982aa488b7f1e7f73e577114f98 +Subproject commit 60244b842c8e4fb7053fd6ada87b22715af389e9 From 2487fe144cdf14c032b8d207383330ffa023e3ad Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 4 Mar 2025 12:26:46 -0800 Subject: [PATCH 4/5] Update .gitmodules and spack for code review and testing --- .gitmodules | 6 ++++-- spack | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.gitmodules b/.gitmodules index 82ae4248d..1a6178d71 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,7 +1,9 @@ [submodule "spack"] path = spack - url = https://github.com/jcsda/spack - branch = release/1.9.0 + #url = https://github.com/jcsda/spack + #branch = release/1.9.0 + url = https://github.com/climbfuji/spack + branch = bugfix/crtm311_strikes_again [submodule "doc/CMakeModules"] path = doc/CMakeModules url = https://github.com/noaa-emc/cmakemodules diff --git a/spack b/spack index 60244b842..66062d7de 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit 60244b842c8e4fb7053fd6ada87b22715af389e9 +Subproject commit 66062d7de57bb475696bea5366b909895420c59d From 9cfe325f6813ff84d2f36a38ba14b0ce8a13900d Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 4 Mar 2025 14:22:19 -0800 Subject: [PATCH 5/5] Revert .gitmodules and update submodule pointer for spack --- .gitmodules | 6 ++---- spack | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.gitmodules b/.gitmodules index 1a6178d71..82ae4248d 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,9 +1,7 @@ [submodule "spack"] path = spack - #url = https://github.com/jcsda/spack - #branch = release/1.9.0 - url = https://github.com/climbfuji/spack - branch = bugfix/crtm311_strikes_again + url = https://github.com/jcsda/spack + branch = release/1.9.0 [submodule "doc/CMakeModules"] path = doc/CMakeModules url = https://github.com/noaa-emc/cmakemodules diff --git a/spack b/spack index 66062d7de..ab2da2468 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit 66062d7de57bb475696bea5366b909895420c59d +Subproject commit ab2da2468530a580a8100faedf79f66f2cf178fd