Add Unstructured WW3 mesh capability to UFS (#1727)

* cpld pdlib, cpld control pass against created baselines * remove "wise" from wave domain name * update WW3, modify rtwaves for testing * set dnats to 0 in pdlib tests * remove scotch from ufs_common and add scotch module only for hera,cheyenne and orion
ufs-community · May 15, 2023 · e403bb4 · e403bb4
1 parent 1ac938c
commit e403bb4
Show file tree

Hide file tree

Showing 27 changed files with 8,384 additions and 8,529 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -70,6 +70,7 @@ set(OPENMP          ON  CACHE BOOL "Enable OpenMP threading")
 set(PARALLEL_NETCDF OFF CACHE BOOL "Enable parallel NetCDF")
 set(JEDI_DRIVER     OFF CACHE BOOL "Enable JEDI as top level driver")
 set(CMEPS_AOFLUX    OFF CACHE BOOL "Enable atmosphere-ocean flux calculation in mediator")
+set(PDLIB           OFF CACHE BOOL "Enable Domain Decomposition in WW3 via PDLIB")
 
 set(CMAKE_Platform $ENV{CMAKE_Platform})
 if(CMAKE_Platform)
@@ -206,7 +207,11 @@ if(WW3)
     set(UFS_CAP "MULTI_ESMF" CACHE STRING "Build with MULTI_ESMF cap")
     list(APPEND _ufs_defs_private FRONT_WW3=WMESMFMD)
   else()
-    set(SWITCH "meshcap" CACHE STRING "NUOPC mesh cap")
+    if(PDLIB)
+      set(SWITCH "meshcap_pdlib" CACHE STRING "NUOPC mesh cap")
+    else()
+      set(SWITCH "meshcap" CACHE STRING "NUOPC mesh cap")
+    endif()
     set(UFS_CAP "NUOPC_MESH" CACHE STRING "Build with NUOPC_MESH cap")
     list(APPEND _ufs_defs_private FRONT_WW3=wav_comp_nuopc)
   endif()

diff --git a/WW3 b/WW3
diff --git a/modulefiles/ufs_cheyenne.gnu.lua b/modulefiles/ufs_cheyenne.gnu.lua
@@ -33,6 +33,9 @@ load(pathJoin("hpc-gnu", hpc_gnu_ver))
 hpc_mpt_ver=os.getenv("hpc_mpt_ver") or "2.25"
 load(pathJoin("hpc-mpt", hpc_mpt_ver))
 
+scotch_ver=os.getenv("scotch_ver") or "7.0.3"
+load(pathJoin("scotch", scotch_ver))
+
 load("ufs_common")
 
 setenv("CC", "mpicc")

diff --git a/modulefiles/ufs_cheyenne.intel.lua b/modulefiles/ufs_cheyenne.intel.lua
@@ -34,6 +34,9 @@ load(pathJoin("hpc-intel", hpc_intel_ver))
 hpc_mpt_ver=os.getenv("hpc_mpt_ver") or "2.25"
 load(pathJoin("hpc-mpt", hpc_mpt_ver))
 
+scotch_ver=os.getenv("scotch_ver") or "7.0.3"
+load(pathJoin("scotch", scotch_ver))
+
 load("ufs_common")
 
 setenv("CC", "mpicc")

diff --git a/modulefiles/ufs_hera.gnu.lua b/modulefiles/ufs_hera.gnu.lua
@@ -22,6 +22,9 @@ load(pathJoin("hpc-gnu", hpc_gnu_ver))
 hpc_mpich_ver=os.getenv("hpc_mpich_ver") or "3.3.2"
 load(pathJoin("hpc-mpich", hpc_mpich_ver))
 
+scotch_ver=os.getenv("scotch_ver") or "7.0.3"
+load(pathJoin("scotch", scotch_ver))
+
 load("ufs_common")
 
 setenv("CC", "mpicc")

diff --git a/modulefiles/ufs_hera.intel.lua b/modulefiles/ufs_hera.intel.lua
@@ -25,6 +25,9 @@ load(pathJoin("hpc-intel", hpc_intel_ver))
 hpc_impi_ver=os.getenv("hpc_impi_ver") or "2022.1.2"
 load(pathJoin("hpc-impi", hpc_impi_ver))
 
+scotch_ver=os.getenv("scotch_ver") or "7.0.3"
+load(pathJoin("scotch", scotch_ver))
+
 load("ufs_common")
 
 setenv("CC", "mpiicc")

diff --git a/modulefiles/ufs_orion.intel.lua b/modulefiles/ufs_orion.intel.lua
@@ -19,6 +19,9 @@ load(pathJoin("hpc-intel", hpc_intel_ver))
 hpc_impi_ver=os.getenv("hpc_impi_ver") or "2022.1.2"
 load(pathJoin("hpc-impi", hpc_impi_ver))
 
+scotch_ver=os.getenv("scotch_ver") or "7.0.3"
+load(pathJoin("scotch", scotch_ver))
+
 load("ufs_common")
 
 setenv("CC", "mpiicc")

diff --git a/tests/RegressionTests_acorn.intel.log b/tests/RegressionTests_acorn.intel.log
diff --git a/tests/RegressionTests_cheyenne.gnu.log b/tests/RegressionTests_cheyenne.gnu.log
diff --git a/tests/RegressionTests_cheyenne.intel.log b/tests/RegressionTests_cheyenne.intel.log
diff --git a/tests/RegressionTests_gaea.intel.log b/tests/RegressionTests_gaea.intel.log
diff --git a/tests/RegressionTests_hera.gnu.log b/tests/RegressionTests_hera.gnu.log
diff --git a/tests/RegressionTests_hera.intel.log b/tests/RegressionTests_hera.intel.log
diff --git a/tests/RegressionTests_jet.intel.log b/tests/RegressionTests_jet.intel.log
diff --git a/tests/RegressionTests_orion.intel.log b/tests/RegressionTests_orion.intel.log
diff --git a/tests/RegressionTests_wcoss2.intel.log b/tests/RegressionTests_wcoss2.intel.log
diff --git a/tests/default_vars.sh b/tests/default_vars.sh
@@ -73,6 +73,18 @@
 
   INPES_aqm=33; JNPES_aqm=8
 
+  THRD_cpl_unstr=1
+  INPES_cpl_unstr=3; JNPES_cpl_unstr=8; WPG_cpl_unstr=6
+  OCN_tasks_cpl_unstr=20
+  ICE_tasks_cpl_unstr=10
+  WAV_tasks_cpl_unstr=60
+
+  THRD_cpl_unstr_mpi=1
+  INPES_cpl_unstr_mpi=4; JNPES_cpl_unstr_mpi=8; WPG_cpl_unstr_mpi=6
+  OCN_tasks_cpl_unstr_mpi=34
+  ICE_tasks_cpl_unstr_mpi=20
+  WAV_tasks_cpl_unstr_mpi=50
+
   aqm_omp_num_threads=1
   atm_omp_num_threads=1
   chm_omp_num_threads=1

diff --git a/tests/parm/nems.configure.cpld_esmfthreads.IN b/tests/parm/nems.configure.cpld_esmfthreads.IN
diff --git a/tests/rt.conf b/tests/rt.conf
@@ -44,6 +44,15 @@ RUN     | cpld_restart_c48
 COMPILE | -DAPP=S2SWA -DCCPP_SUITES=FV3_GFS_v17_coupled_p8,FV3_GFS_cpld_rasmgshocnsstnoahmp_ugwp -DFASTER=ON                      | - cheyenne.intel                        | fv3 |
 RUN     | cpld_control_p8_faster                                                                                                  | - cheyenne.intel                        | fv3 |
 
+# Unstructured WW3 mesh
+COMPILE | -DAPP=S2SW -DCCPP_SUITES=FV3_GFS_v17_coupled_p8 -DPDLIB=ON                                                              | +hera.intel orion.intel cheyenne.intel  | fv3 |
+RUN     | cpld_control_pdlib_p8                                                                                                   | +hera.intel orion.intel cheyenne.intel  | fv3 |
+RUN     | cpld_restart_pdlib_p8                                                                                                   | +hera.intel orion.intel cheyenne.intel  |     | cpld_control_pdlib_p8
+RUN     | cpld_mpi_pdlib_p8                                                                                                       | +hera.intel orion.intel cheyenne.intel  |     | cpld_control_pdlib_p8
+
+COMPILE | -DAPP=S2SW -DCCPP_SUITES=FV3_GFS_v17_coupled_p8 -DPDLIB=ON -DDEBUG=ON                                                   | +hera.intel orion.intel cheyenne.intel  | fv3 |
+RUN     | cpld_debug_pdlib_p8                                                                                                     | +hera.intel orion.intel cheyenne.intel  | fv3 |
+
 ###################################################################################################################################################################################
 # ATM tests                                                                                                                                                                       #
 ###################################################################################################################################################################################

diff --git a/tests/rt.sh b/tests/rt.sh
@@ -358,12 +358,12 @@ elif [[ $MACHINE_ID = expanse.* ]]; then
   PTMP=$dprefix
   SCHEDULER=slurm
   cp fv3_conf/fv3_slurm.IN_expanse fv3_conf/fv3_slurm.IN
-  
+
  elif [[ $MACHINE_ID = noaacloud.* ]]; then
-   
+
   module use /apps/modules/modulefiles
   module load rocoto/1.3.3
-   
+
   ROCOTORUN=$(which rocotorun)
   ROCOTOSTAT=$(which rocotostat)
   ROCOTOCOMPLETE=$(which rocotocomplete)
@@ -379,8 +379,8 @@ elif [[ $MACHINE_ID = expanse.* ]]; then
   SCHEDULER=slurm
   cp fv3_conf/fv3_slurm.IN_noaacloud fv3_conf/fv3_slurm.IN
   cp fv3_conf/compile_slurm.IN_noaacloud fv3_conf/compile_slurm.IN
-  
-  
+
+
 else
   die "Unknown machine ID, please edit detect_machine.sh file"
 fi
@@ -468,7 +468,7 @@ if [[ $TESTS_FILE =~ '35d' ]] || [[ $TESTS_FILE =~ 'weekly' ]]; then
 fi
 
 
-BL_DATE=20230510
+BL_DATE=20230512
 
 RTPWD=${RTPWD:-$DISKNM/NEMSfv3gfs/develop-${BL_DATE}/${RT_COMPILER^^}}
 

diff --git a/tests/rt_gnu.conf b/tests/rt_gnu.conf
@@ -95,14 +95,21 @@ RUN     | rap_control_dyn64_phy32_debug
 # S2S tests                                                                                                                                                      #
 ##################################################################################################################################################################
 
-COMPILE | -DAPP=S2SWA -DCCPP_SUITES=FV3_GFS_v17_coupled_p8,FV3_GFS_cpld_rasmgshocnsstnoahmp_ugwp                                   |                | fv3         |
-RUN     | cpld_control_p8                                                                                                          |                | fv3         |
+COMPILE | -DAPP=S2SWA -DCCPP_SUITES=FV3_GFS_v17_coupled_p8,FV3_GFS_cpld_rasmgshocnsstnoahmp_ugwp                                   |                | fv3 |
+RUN     | cpld_control_p8                                                                                                          |                | fv3 |
 
-COMPILE | -DAPP=S2S -DCCPP_SUITES=FV3_GFS_v17_coupled_p8,FV3_GFS_cpld_rasmgshocnsstnoahmp_ugwp                                     |                | fv3         |
-RUN     | cpld_control_nowave_noaero_p8                                                                                            |                | fv3         |
+COMPILE | -DAPP=S2S -DCCPP_SUITES=FV3_GFS_v17_coupled_p8,FV3_GFS_cpld_rasmgshocnsstnoahmp_ugwp                                     |                | fv3 |
+RUN     | cpld_control_nowave_noaero_p8                                                                                            |                | fv3 |
 
-COMPILE | -DAPP=S2SWA -DDEBUG=ON -DCCPP_SUITES=FV3_GFS_v17_coupled_p8,FV3_GFS_cpld_rasmgshocnsstnoahmp_ugwp                        |                | fv3         |
-RUN     | cpld_debug_p8                                                                                                            |                | fv3         |
+COMPILE | -DAPP=S2SWA -DDEBUG=ON -DCCPP_SUITES=FV3_GFS_v17_coupled_p8,FV3_GFS_cpld_rasmgshocnsstnoahmp_ugwp                        |                | fv3 |
+RUN     | cpld_debug_p8                                                                                                            |                | fv3 |
+
+# Unstructured WW3
+COMPILE | -DAPP=S2SW -DCCPP_SUITES=FV3_GFS_v17_coupled_p8 -DPDLIB=ON                                                              |                 | fv3 |
+RUN     | cpld_control_pdlib_p8                                                                                                   |                 | fv3 |
+
+COMPILE | -DAPP=S2SW -DCCPP_SUITES=FV3_GFS_v17_coupled_p8 -DPDLIB=ON -DDEBUG=ON                                                   |                 | fv3 |
+RUN     | cpld_debug_pdlib_p8                                                                                                     |                 | fv3 |
 
 ##################################################################################################################################################################
 # CDEPS Data Atmosphere test                                                                                                                                     #

diff --git a/tests/tests/cpld_bmark_p8 b/tests/tests/cpld_bmark_p8
@@ -54,7 +54,7 @@ export LIST_FILES="sfcf006.nc \
 export_fv3
 export_cpl
 
-export NEMS_CONFIGURE=nems.configure.cpld_esmfthreads.IN
+export NEMS_CONFIGURE=nems.configure.cpld.IN
 
 export SYEAR=2013
 export SMONTH=04