Error building with nvidia/22.7, cuda11.7, and kokkos 4.0.01 on Perlmutter

[zhangchonglin]

While building omega_h master branch at latest commit e1be29b on Perlmutter with the following libraries:

• nvidia 22.7
• cuda 11.7
• and kokkos 4.0.01

I encountered the following error:

-- Cray Programming Environment 2.7.20 CXX
-- CMAKE_VERSION: 3.22.0
-- Omega_h_VERSION: 10.5.0
-- USE_XSDK_DEFAULTS: OFF
-- BUILD_TESTING: OFF
-- BUILD_SHARED_LIBS: OFF
-- CMAKE_INSTALL_PREFIX: /global/homes/z/zhangc20/xgcm/install_nvhpc22.7/omega_h/install
-- Omega_h_CHECK_BOUNDS: OFF
-- Omega_h_THROW: OFF
-- Omega_h_DATA: 
-- Omega_h_USE_EGADS: OFF
-- EGADS_PREFIX: 
-- Omega_h_USE_Kokkos: ON
-- Kokkos_PREFIX: /global/homes/z/zhangc20/xgcm/install_nvhpc22.7/kokkos/install/lib64/cmake
-- Omega_h_USE_CUDA_AWARE_MPI: OFF
-- Omega_h_USE_SimModSuite: OFF
-- Omega_h_VALGRIND: 
-- Omega_h_EXAMPLES: OFF
-- Omega_h_USE_MPI: ON
-- Omega_h_ENABLE_DEMANGLED_STACKTRACE: OFF
-- Omega_h_DBG: OFF
-- Cray Programming Environment 2.7.20 C
-- Omega_h_USE_MPI: ON
-- MPI_PREFIX: 
-- Omega_h_USE_ZLIB: ON
-- ZLIB_PREFIX: 
-- Omega_h_USE_Kokkos: ON
-- Kokkos_PREFIX: /global/homes/z/zhangc20/xgcm/install_nvhpc22.7/kokkos/install/lib64/cmake
-- Enabled Kokkos devices: CUDA;SERIAL
-- kokkos_launch_compiler is enabled globally. C++ compiler commands with -DKOKKOS_DEPENDENCE will be redirected to the appropriate compiler for Kokkos
-- Kokkos_CONFIG: /global/homes/z/zhangc20/xgcm/install_nvhpc22.7/kokkos/install/lib64/cmake/Kokkos/KokkosConfig.cmake
-- Kokkos_VERSION: 4.0.1
-- Omega_h_USE_libMeshb: OFF
-- Omega_h_USE_Gmsh: OFF
-- Omega_h_USE_Gmodel: OFF
-- Omega_h_USE_SEACASExodus: OFF
-- Omega_h_USE_pybind11: OFF
-- Omega_h_USE_CUDA: on
-- Omega_h_USE_MPI: ON
-- Omega_h_USE_OpenMP: OFF
-- Omega_h_USE_CUDA: on
-- Omega_h_USE_DOLFIN: OFF
-- Omega_h_SEMVER = 10.5.0-sha.e1be29b0+100101100000000000
-- Configuring done
-- Generating done
-- Build files have been written to: /global/homes/z/zhangc20/xgcm/install_nvhpc22.7/omega_h/build
/global/common/software/nersc/pm-2021q4/sw/cmake-3.22.0/bin/cmake -S/global/homes/z/zhangc20/xgcm/omega_h -B/global/homes/z/zhangc20/xgcm/install_nvhpc22.7/omega_h/build --check-build-system CMakeFiles/Makefile.cmake 0
/global/common/software/nersc/pm-2021q4/sw/cmake-3.22.0/bin/cmake -E cmake_progress_start /global/homes/z/zhangc20/xgcm/install_nvhpc22.7/omega_h/build/CMakeFiles /global/homes/z/zhangc20/xgcm/install_nvhpc22.7/omega_h/build//CMakeFiles/progress.marks
make  -f CMakeFiles/Makefile2 all
make[1]: Entering directory '/global/u2/z/zhangc20/xgcm/install_nvhpc22.7/omega_h/build'
make  -f src/CMakeFiles/omega_h.dir/build.make src/CMakeFiles/omega_h.dir/depend
make[2]: Entering directory '/global/u2/z/zhangc20/xgcm/install_nvhpc22.7/omega_h/build'
cd /global/homes/z/zhangc20/xgcm/install_nvhpc22.7/omega_h/build && /global/common/software/nersc/pm-2021q4/sw/cmake-3.22.0/bin/cmake -E cmake_depends "Unix Makefiles" /global/homes/z/zhangc20/xgcm/omega_h /global/homes/z/zhangc20/xgcm/omega_h/src /global/homes/z/zhangc20/xgcm/install_nvhpc22.7/omega_h/build /global/homes/z/zhangc20/xgcm/install_nvhpc22.7/omega_h/build/src /global/homes/z/zhangc20/xgcm/install_nvhpc22.7/omega_h/build/src/CMakeFiles/omega_h.dir/DependInfo.cmake --color=
make[2]: Leaving directory '/global/u2/z/zhangc20/xgcm/install_nvhpc22.7/omega_h/build'
make  -f src/CMakeFiles/omega_h.dir/build.make src/CMakeFiles/omega_h.dir/build
make[2]: Entering directory '/global/u2/z/zhangc20/xgcm/install_nvhpc22.7/omega_h/build'
[  1%] Building CXX object src/CMakeFiles/omega_h.dir/Omega_h_adapt.cpp.o
cd /global/homes/z/zhangc20/xgcm/install_nvhpc22.7/omega_h/build/src && /global/homes/z/zhangc20/xgcm/install_nvhpc22.7/kokkos/install/bin/kokkos_launch_compiler /opt/cray/pe/craype/2.7.20/bin/CC /opt/cray/pe/craype/2.7.20/bin/CC /opt/cray/pe/craype/2.7.20/bin/CC -DKOKKOS_DEPENDENCE -I/global/homes/z/zhangc20/xgcm/omega_h/src -I/global/homes/z/zhangc20/xgcm/install_nvhpc22.7/omega_h/build/src -I/global/homes/z/zhangc20/xgcm/omega_h/tpl -isystem /global/homes/z/zhangc20/xgcm/install_nvhpc22.7/kokkos/install/include -isystem /opt/nvidia/hpc_sdk/Linux_x86_64/22.7/cuda/11.7/include -std=c++14 -fast -O3 -DNDEBUG --expt-extended-lambda -gpu=nordc -cuda -gpu=cc80 --c++17 -MD -MT src/CMakeFiles/omega_h.dir/Omega_h_adapt.cpp.o -MF CMakeFiles/omega_h.dir/Omega_h_adapt.cpp.o.d -o CMakeFiles/omega_h.dir/Omega_h_adapt.cpp.o -c /global/homes/z/zhangc20/xgcm/omega_h/src/Omega_h_adapt.cpp
[  2%] Building CXX object src/CMakeFiles/omega_h.dir/Omega_h_adj.cpp.o
[  2%] Building CXX object src/CMakeFiles/omega_h.dir/Omega_h_align.cpp.o
cd /global/homes/z/zhangc20/xgcm/install_nvhpc22.7/omega_h/build/src && /global/homes/z/zhangc20/xgcm/install_nvhpc22.7/kokkos/install/bin/kokkos_launch_compiler /opt/cray/pe/craype/2.7.20/bin/CC /opt/cray/pe/craype/2.7.20/bin/CC /opt/cray/pe/craype/2.7.20/bin/CC -DKOKKOS_DEPENDENCE -I/global/homes/z/zhangc20/xgcm/omega_h/src -I/global/homes/z/zhangc20/xgcm/install_nvhpc22.7/omega_h/build/src -I/global/homes/z/zhangc20/xgcm/omega_h/tpl -isystem /global/homes/z/zhangc20/xgcm/install_nvhpc22.7/kokkos/install/include -isystem /opt/nvidia/hpc_sdk/Linux_x86_64/22.7/cuda/11.7/include -std=c++14 -fast -O3 -DNDEBUG --expt-extended-lambda -gpu=nordc -cuda -gpu=cc80 --c++17 -MD -MT src/CMakeFiles/omega_h.dir/Omega_h_align.cpp.o -MF CMakeFiles/omega_h.dir/Omega_h_align.cpp.o.d -o CMakeFiles/omega_h.dir/Omega_h_align.cpp.o -c /global/homes/z/zhangc20/xgcm/omega_h/src/Omega_h_align.cpp
cd /global/homes/z/zhangc20/xgcm/install_nvhpc22.7/omega_h/build/src && /global/homes/z/zhangc20/xgcm/install_nvhpc22.7/kokkos/install/bin/kokkos_launch_compiler /opt/cray/pe/craype/2.7.20/bin/CC /opt/cray/pe/craype/2.7.20/bin/CC /opt/cray/pe/craype/2.7.20/bin/CC -DKOKKOS_DEPENDENCE -I/global/homes/z/zhangc20/xgcm/omega_h/src -I/global/homes/z/zhangc20/xgcm/install_nvhpc22.7/omega_h/build/src -I/global/homes/z/zhangc20/xgcm/omega_h/tpl -isystem /global/homes/z/zhangc20/xgcm/install_nvhpc22.7/kokkos/install/include -isystem /opt/nvidia/hpc_sdk/Linux_x86_64/22.7/cuda/11.7/include -std=c++14 -fast -O3 -DNDEBUG --expt-extended-lambda -gpu=nordc -cuda -gpu=cc80 --c++17 -MD -MT src/CMakeFiles/omega_h.dir/Omega_h_adj.cpp.o -MF CMakeFiles/omega_h.dir/Omega_h_adj.cpp.o.d -o CMakeFiles/omega_h.dir/Omega_h_adj.cpp.o -c /global/homes/z/zhangc20/xgcm/omega_h/src/Omega_h_adj.cpp
[  4%] Building CXX object src/CMakeFiles/omega_h.dir/Omega_h_amr.cpp.o
[  4%] Building CXX object src/CMakeFiles/omega_h.dir/Omega_h_any.cpp.o
cd /global/homes/z/zhangc20/xgcm/install_nvhpc22.7/omega_h/build/src && /global/homes/z/zhangc20/xgcm/install_nvhpc22.7/kokkos/install/bin/kokkos_launch_compiler /opt/cray/pe/craype/2.7.20/bin/CC /opt/cray/pe/craype/2.7.20/bin/CC /opt/cray/pe/craype/2.7.20/bin/CC -DKOKKOS_DEPENDENCE -I/global/homes/z/zhangc20/xgcm/omega_h/src -I/global/homes/z/zhangc20/xgcm/install_nvhpc22.7/omega_h/build/src -I/global/homes/z/zhangc20/xgcm/omega_h/tpl -isystem /global/homes/z/zhangc20/xgcm/install_nvhpc22.7/kokkos/install/include -isystem /opt/nvidia/hpc_sdk/Linux_x86_64/22.7/cuda/11.7/include -std=c++14 -fast -O3 -DNDEBUG --expt-extended-lambda -gpu=nordc -cuda -gpu=cc80 --c++17 -MD -MT src/CMakeFiles/omega_h.dir/Omega_h_amr.cpp.o -MF CMakeFiles/omega_h.dir/Omega_h_amr.cpp.o.d -o CMakeFiles/omega_h.dir/Omega_h_amr.cpp.o -c /global/homes/z/zhangc20/xgcm/omega_h/src/Omega_h_amr.cpp
cd /global/homes/z/zhangc20/xgcm/install_nvhpc22.7/omega_h/build/src && /global/homes/z/zhangc20/xgcm/install_nvhpc22.7/kokkos/install/bin/kokkos_launch_compiler /opt/cray/pe/craype/2.7.20/bin/CC /opt/cray/pe/craype/2.7.20/bin/CC /opt/cray/pe/craype/2.7.20/bin/CC -DKOKKOS_DEPENDENCE -I/global/homes/z/zhangc20/xgcm/omega_h/src -I/global/homes/z/zhangc20/xgcm/install_nvhpc22.7/omega_h/build/src -I/global/homes/z/zhangc20/xgcm/omega_h/tpl -isystem /global/homes/z/zhangc20/xgcm/install_nvhpc22.7/kokkos/install/include -isystem /opt/nvidia/hpc_sdk/Linux_x86_64/22.7/cuda/11.7/include -std=c++14 -fast -O3 -DNDEBUG --expt-extended-lambda -gpu=nordc -cuda -gpu=cc80 --c++17 -MD -MT src/CMakeFiles/omega_h.dir/Omega_h_any.cpp.o -MF CMakeFiles/omega_h.dir/Omega_h_any.cpp.o.d -o CMakeFiles/omega_h.dir/Omega_h_any.cpp.o -c /global/homes/z/zhangc20/xgcm/omega_h/src/Omega_h_any.cpp
[  5%] Building CXX object src/CMakeFiles/omega_h.dir/Omega_h_amr_transfer.cpp.o
[  5%] Building CXX object src/CMakeFiles/omega_h.dir/Omega_h_approach.cpp.o
cd /global/homes/z/zhangc20/xgcm/install_nvhpc22.7/omega_h/build/src && /global/homes/z/zhangc20/xgcm/install_nvhpc22.7/kokkos/install/bin/kokkos_launch_compiler /opt/cray/pe/craype/2.7.20/bin/CC /opt/cray/pe/craype/2.7.20/bin/CC /opt/cray/pe/craype/2.7.20/bin/CC -DKOKKOS_DEPENDENCE -I/global/homes/z/zhangc20/xgcm/omega_h/src -I/global/homes/z/zhangc20/xgcm/install_nvhpc22.7/omega_h/build/src -I/global/homes/z/zhangc20/xgcm/omega_h/tpl -isystem /global/homes/z/zhangc20/xgcm/install_nvhpc22.7/kokkos/install/include -isystem /opt/nvidia/hpc_sdk/Linux_x86_64/22.7/cuda/11.7/include -std=c++14 -fast -O3 -DNDEBUG --expt-extended-lambda -gpu=nordc -cuda -gpu=cc80 --c++17 -MD -MT src/CMakeFiles/omega_h.dir/Omega_h_approach.cpp.o -MF CMakeFiles/omega_h.dir/Omega_h_approach.cpp.o.d -o CMakeFiles/omega_h.dir/Omega_h_approach.cpp.o -c /global/homes/z/zhangc20/xgcm/omega_h/src/Omega_h_approach.cpp
cd /global/homes/z/zhangc20/xgcm/install_nvhpc22.7/omega_h/build/src && /global/homes/z/zhangc20/xgcm/install_nvhpc22.7/kokkos/install/bin/kokkos_launch_compiler /opt/cray/pe/craype/2.7.20/bin/CC /opt/cray/pe/craype/2.7.20/bin/CC /opt/cray/pe/craype/2.7.20/bin/CC -DKOKKOS_DEPENDENCE -I/global/homes/z/zhangc20/xgcm/omega_h/src -I/global/homes/z/zhangc20/xgcm/install_nvhpc22.7/omega_h/build/src -I/global/homes/z/zhangc20/xgcm/omega_h/tpl -isystem /global/homes/z/zhangc20/xgcm/install_nvhpc22.7/kokkos/install/include -isystem /opt/nvidia/hpc_sdk/Linux_x86_64/22.7/cuda/11.7/include -std=c++14 -fast -O3 -DNDEBUG --expt-extended-lambda -gpu=nordc -cuda -gpu=cc80 --c++17 -MD -MT src/CMakeFiles/omega_h.dir/Omega_h_amr_transfer.cpp.o -MF CMakeFiles/omega_h.dir/Omega_h_amr_transfer.cpp.o.d -o CMakeFiles/omega_h.dir/Omega_h_amr_transfer.cpp.o -c /global/homes/z/zhangc20/xgcm/omega_h/src/Omega_h_amr_transfer.cpp
[  6%] Building CXX object src/CMakeFiles/omega_h.dir/Omega_h_amr_topology.cpp.o
cd /global/homes/z/zhangc20/xgcm/install_nvhpc22.7/omega_h/build/src && /global/homes/z/zhangc20/xgcm/install_nvhpc22.7/kokkos/install/bin/kokkos_launch_compiler /opt/cray/pe/craype/2.7.20/bin/CC /opt/cray/pe/craype/2.7.20/bin/CC /opt/cray/pe/craype/2.7.20/bin/CC -DKOKKOS_DEPENDENCE -I/global/homes/z/zhangc20/xgcm/omega_h/src -I/global/homes/z/zhangc20/xgcm/install_nvhpc22.7/omega_h/build/src -I/global/homes/z/zhangc20/xgcm/omega_h/tpl -isystem /global/homes/z/zhangc20/xgcm/install_nvhpc22.7/kokkos/install/include -isystem /opt/nvidia/hpc_sdk/Linux_x86_64/22.7/cuda/11.7/include -std=c++14 -fast -O3 -DNDEBUG --expt-extended-lambda -gpu=nordc -cuda -gpu=cc80 --c++17 -MD -MT src/CMakeFiles/omega_h.dir/Omega_h_amr_topology.cpp.o -MF CMakeFiles/omega_h.dir/Omega_h_amr_topology.cpp.o.d -o CMakeFiles/omega_h.dir/Omega_h_amr_topology.cpp.o -c /global/homes/z/zhangc20/xgcm/omega_h/src/Omega_h_amr_topology.cpp
nvc++-Error-Unknown switch: --expt-extended-lambda
make[2]: *** [src/CMakeFiles/omega_h.dir/build.make:90: src/CMakeFiles/omega_h.dir/Omega_h_adj.cpp.o] Error 1
make[2]: *** Waiting for unfinished jobs....
nvc++-Error-Unknown switch: --expt-extended-lambda
nvc++-Error-nvc++-Error-Unknown switch: --expt-extended-lambdaUnknown switch: --expt-extended-lambda

nvc++-Error-Unknown switch: --expt-extended-lambda
nvc++-Error-Unknown switch: --expt-extended-lambda
make[2]: *** [src/CMakeFiles/omega_h.dir/build.make:174: src/CMakeFiles/omega_h.dir/Omega_h_approach.cpp.o] Error 1
make[2]: *** [src/CMakeFiles/omega_h.dir/build.make:160: src/CMakeFiles/omega_h.dir/Omega_h_any.cpp.o] Error 1
make[2]: *** [src/CMakeFiles/omega_h.dir/build.make:104: src/CMakeFiles/omega_h.dir/Omega_h_align.cpp.o] Error 1
nvc++-Error-make[2]: *** [src/CMakeFiles/omega_h.dir/build.make:76: src/CMakeFiles/omega_h.dir/Omega_h_adapt.cpp.o] Error 1
Unknown switch: --expt-extended-lambda
make[2]: *** [src/CMakeFiles/omega_h.dir/build.make:132: src/CMakeFiles/omega_h.dir/Omega_h_amr_topology.cpp.o] Error 1
make[2]: *** [src/CMakeFiles/omega_h.dir/build.make:118: src/CMakeFiles/omega_h.dir/Omega_h_amr.cpp.o] Error 1
nvc++-Error-Unknown switch: --expt-extended-lambda
make[2]: *** [src/CMakeFiles/omega_h.dir/build.make:146: src/CMakeFiles/omega_h.dir/Omega_h_amr_transfer.cpp.o] Error 1
make[2]: Leaving directory '/global/u2/z/zhangc20/xgcm/install_nvhpc22.7/omega_h/build'
make[1]: *** [CMakeFiles/Makefile2:128: src/CMakeFiles/omega_h.dir/all] Error 2
make[1]: Leaving directory '/global/u2/z/zhangc20/xgcm/install_nvhpc22.7/omega_h/build'
make: *** [Makefile:146: all] Error 2

The configuration and build script:

module load PrgEnv-nvidia                                                       
module load nvidia/22.7                                                            
module load cudatoolkit/11.7                                                       
module load craype-accel-nvidia80                                               
module load cmake/3.22.0                                                        
module unload darshan                                                           
                                                                                
export cuda=$CRAY_CUDATOOLKIT_DIR                                               
export PATH=$cuda/bin:$PATH                                                     
export LD_LIBRARY_PATH=$cuda/lib64:$LD_LIBRARY_PATH                             
export installroot=$PWD                                                         
export kk=$installroot/kokkos/install                                           
export srcroot=$installroot/../                                                 
export kksrc=$srcroot/kokkos                                                    
                                                                                
export oh=$installroot/omega_h/install                                          
export ohsrc=$srcroot/omega_h                                                   
                                                                                
cd $installroot                                                                 
mkdir -p omega_h/build                                                          
cd omega_h/build                                                                
cmake $ohsrc -DCMAKE_INSTALL_PREFIX=$oh -DBUILD_SHARED_LIBS=OFF \               
             -DCMAKE_BUILD_TYPE=Release \                                       
             -DOmega_h_USE_CUDA=on -DOmega_h_CUDA_ARCH=80 \                     
             -DOmega_h_USE_Kokkos=ON -DOmega_h_USE_MPI=ON \                     
             -DCMAKE_CXX_COMPILER=CC \                                          
             -DCMAKE_CXX_FLAGS='-std=c++14' \                                   
             -DCMAKE_CUDA_FLAGS="-I$MPICH_DIR/include" \                        
             -DKokkos_PREFIX=$kk/lib64/cmake/                                   
                                                                                
make VERBOSE=1 -j8 install     

Previously, I was able to build omega_h using gcc 11.2.0, cuda 11.7, and kokkos 4.0.01 on Perlmutter.