ufs-community · jkbk2004 · Jul 19, 2022 · May 2, 2022 · May 2, 2022 · May 3, 2022
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -54,7 +54,8 @@ message("CMEPS ............ ${CMEPS}")
 ###############################################################################
 ### Build Options
 ###############################################################################
-set(32BIT           OFF CACHE BOOL "Enable 32BIT (single precision arithmetic in dycore)")
+set(32BIT           OFF CACHE BOOL "Enable 32BIT (single precision arithmetic in dycore and fast physics)")
+set(CCPP_32BIT      OFF CACHE BOOL "Enable CCPP_32BIT (single precision arithmetic in slow physics)")
 set(AVX2            ON  CACHE BOOL "Enable AVX2 instruction set")
 set(AVX             OFF CACHE BOOL "Enable AVX-I instruction set")
 set(SIMDMULTIARCH   OFF CACHE BOOL "Enable multi-target SIMD instruction sets")
@@ -81,6 +82,7 @@ endif()
 
 message("")
 message("32BIT ............ ${32BIT}")
+message("CCPP_32BIT ....... ${CCPP_32BIT}")
 message("AVX2 ............. ${AVX2}")
 message("SIMDMULTIARCH .... ${SIMDMULTIARCH}")
 message("DEBUG ............ ${DEBUG}")

diff --git a/FV3 b/FV3
diff --git a/stochastic_physics b/stochastic_physics
diff --git a/tests/RegressionTests_cheyenne.gnu.log b/tests/RegressionTests_cheyenne.gnu.log
diff --git a/tests/RegressionTests_cheyenne.intel.log b/tests/RegressionTests_cheyenne.intel.log
diff --git a/tests/RegressionTests_gaea.intel.log b/tests/RegressionTests_gaea.intel.log
diff --git a/tests/RegressionTests_hera.gnu.log b/tests/RegressionTests_hera.gnu.log
diff --git a/tests/RegressionTests_hera.intel.log b/tests/RegressionTests_hera.intel.log
diff --git a/tests/RegressionTests_jet.intel.log b/tests/RegressionTests_jet.intel.log
diff --git a/tests/RegressionTests_orion.intel.log b/tests/RegressionTests_orion.intel.log
diff --git a/tests/RegressionTests_wcoss2.intel.log b/tests/RegressionTests_wcoss2.intel.log
diff --git a/tests/rt.conf b/tests/rt.conf
@@ -143,6 +143,31 @@ RUN     | rrfs_v1beta_debug
 COMPILE | -DAPP=ATM -DCCPP_SUITES=FV3_GFS_v16_fv3wam -D32BIT=ON -DMULTI_GASES=ON -DDEBUG=ON                                       |                                         | fv3 |
 RUN     | control_wam_debug                                                                                                       |                                         | fv3 |
 
+###################################################################################################################################################################################
+# 32-bit physics tests                                                                                                                                                            #
+###################################################################################################################################################################################
+#
+
+COMPILE | -DAPP=ATM -DCCPP_SUITES=FV3_RAP,FV3_HRRR -D32BIT=ON -DCCPP_32BIT=ON                                                     |                | fv3 |
+RUN     | regional_spp_sppt_shum_skeb_dyn32_phy32                                                                                 |                | fv3 |
+RUN     | rap_control_dyn32_phy32                                                                                                 |                | fv3 |
+RUN     | hrrr_control_dyn32_phy32                                                                                                |                | fv3 |
+RUN     | rap_2threads_dyn32_phy32                                                                                                | - wcoss_cray   |     |
+RUN     | hrrr_control_2threads_dyn32_phy32                                                                                       | - wcoss_cray   |     |
+RUN     | hrrr_control_decomp_dyn32_phy32                                                                                         |                |     |
+RUN     | rap_restart_dyn32_phy32                                                                                                 |                |     | rap_control_dyn32_phy32
+RUN     | hrrr_control_restart_dyn32_phy32                                                                                        |                |     | hrrr_control_dyn32_phy32
+
+COMPILE | -DAPP=ATM -DCCPP_SUITES=FV3_RAP,FV3_HRRR -DCCPP_32BIT=ON                                                                |                | fv3 |
+RUN     | rap_control_dyn64_phy32                                                                                                 |                | fv3 |
+
+COMPILE | -DAPP=ATM -DCCPP_SUITES=FV3_RAP,FV3_HRRR -D32BIT=ON -DCCPP_32BIT=ON -DDEBUG=ON                                          |                | fv3 |
+RUN     | rap_control_debug_dyn32_phy32                                                                                           |                | fv3 |
+RUN     | hrrr_control_debug_dyn32_phy32                                                                                          |                | fv3 |
+
+COMPILE | -DAPP=ATM -DCCPP_SUITES=FV3_RAP,FV3_HRRR -DCCPP_32BIT=ON -DDEBUG=ON                                                     |                | fv3 |
+RUN     | rap_control_dyn64_phy32_debug                                                                                           |                | fv3 |
+
 ###################################################################################################################################################################################
 # HAFS tests                                                                                                                                                                      #
 ###################################################################################################################################################################################

diff --git a/tests/rt_gnu.conf b/tests/rt_gnu.conf
@@ -51,6 +51,30 @@ RUN     | control_debug_p8
 COMPILE | -DAPP=ATM -DCCPP_SUITES=FV3_GFS_v16_fv3wam -D32BIT=ON -DMULTI_GASES=ON -DDEBUG=ON                                       |                | fv3         |
 RUN     | control_wam_debug                                                                                                       |                | fv3         |
 
+
+##################################################################################################################################################################
+# 32-bit physics tests                                                                                                                                           #
+##################################################################################################################################################################
+
+COMPILE | -DAPP=ATM -DCCPP_SUITES=FV3_RAP,FV3_HRRR -D32BIT=ON -DCCPP_32BIT=ON                                                     |                | fv3 |
+RUN     | rap_control_dyn32_phy32                                                                                                 |                | fv3 |
+RUN     | hrrr_control_dyn32_phy32                                                                                                |                | fv3 |
+RUN     | rap_2threads_dyn32_phy32                                                                                                |                |     |
+RUN     | hrrr_control_2threads_dyn32_phy32                                                                                       |                |     |
+RUN     | hrrr_control_decomp_dyn32_phy32                                                                                         |                |     |
+RUN     | rap_restart_dyn32_phy32                                                                                                 |                |     | rap_control_dyn32_phy32
+RUN     | hrrr_control_restart_dyn32_phy32                                                                                        |                |     | hrrr_control_dyn32_phy32
+
+COMPILE | -DAPP=ATM -DCCPP_32BIT=ON                                                                                               |                | fv3 |
+RUN     | rap_control_dyn64_phy32                                                                                                 |                | fv3 |
+
+COMPILE | -DAPP=ATM -D32BIT=ON -DCCPP_32BIT=ON -DDEBUG=ON                                                                         |                | fv3 |
+RUN     | rap_control_debug_dyn32_phy32                                                                                           |                | fv3 |
+RUN     | hrrr_control_debug_dyn32_phy32                                                                                          |                | fv3 |
+
+COMPILE | -DAPP=ATM -DCCPP_32BIT=ON -DDEBUG=ON                                                                                    |                | fv3 |
+RUN     | rap_control_dyn64_phy32_debug                                                                                           |                | fv3 |
+
 ##################################################################################################################################################################
 # S2S tests                                                                                                                                                      #
 ##################################################################################################################################################################

diff --git a/tests/rt_utils.sh b/tests/rt_utils.sh
@@ -466,11 +466,6 @@ rocoto_step() {
     #   Is it done?
     state=$($ROCOTOSTAT -w $ROCOTO_XML -d $ROCOTO_DB -s | grep 197001010000 | awk -F" " '{print $2}')
     echo "$state" > $ROCOTO_STATE
-    dead_compile=$($ROCOTOSTAT -w $ROCOTO_XML -d $ROCOTO_DB | grep compile_ | grep DEAD | head -1 | awk -F" " '{print $2}')
-    if [[ ! -z ${dead_compile} ]]; then
-        echo "y" | ${ROCOTOCOMPLETE} -w $ROCOTO_XML -d $ROCOTO_DB -m ${dead_compile}_tasks
-        ${ROCOTOCOMPLETE} -w $ROCOTO_XML -d $ROCOTO_DB -t ${dead_compile}
-    fi
 }
 
 rocoto_run() {

diff --git a/tests/tests/hrrr_control_2threads_dyn32_phy32 b/tests/tests/hrrr_control_2threads_dyn32_phy32
@@ -0,0 +1,114 @@
+###############################################################################
+#
+#  HRRR control with 2 threads test
+#
+###############################################################################
+
+export TEST_DESCR="Compare HRRR with 2 threads results with control test"
+
+export CNTL_DIR=hrrr_control_dyn32_phy32
+
+export LIST_FILES="sfcf000.nc \
+                   sfcf009.nc \
+                   sfcf012.nc \
+                   atmf000.nc \
+                   atmf009.nc \
+                   atmf012.nc \
+                   GFSFLX.GrbF00 \
+                   GFSFLX.GrbF09 \
+                   GFSFLX.GrbF12 \
+                   GFSPRS.GrbF00 \
+                   GFSPRS.GrbF09 \
+                   GFSPRS.GrbF12 \
+                   RESTART/coupler.res \
+                   RESTART/fv_core.res.nc \
+                   RESTART/fv_core.res.tile1.nc \
+                   RESTART/fv_core.res.tile2.nc \
+                   RESTART/fv_core.res.tile3.nc \
+                   RESTART/fv_core.res.tile4.nc \
+                   RESTART/fv_core.res.tile5.nc \
+                   RESTART/fv_core.res.tile6.nc \
+                   RESTART/fv_srf_wnd.res.tile1.nc \
+                   RESTART/fv_srf_wnd.res.tile2.nc \
+                   RESTART/fv_srf_wnd.res.tile3.nc \
+                   RESTART/fv_srf_wnd.res.tile4.nc \
+                   RESTART/fv_srf_wnd.res.tile5.nc \
+                   RESTART/fv_srf_wnd.res.tile6.nc \
+                   RESTART/fv_tracer.res.tile1.nc \
+                   RESTART/fv_tracer.res.tile2.nc \
+                   RESTART/fv_tracer.res.tile3.nc \
+                   RESTART/fv_tracer.res.tile4.nc \
+                   RESTART/fv_tracer.res.tile5.nc \
+                   RESTART/fv_tracer.res.tile6.nc \
+                   RESTART/phy_data.tile1.nc \
+                   RESTART/phy_data.tile2.nc \
+                   RESTART/phy_data.tile3.nc \
+                   RESTART/phy_data.tile4.nc \
+                   RESTART/phy_data.tile5.nc \
+                   RESTART/phy_data.tile6.nc \
+                   RESTART/sfc_data.tile1.nc \
+                   RESTART/sfc_data.tile2.nc \
+                   RESTART/sfc_data.tile3.nc \
+                   RESTART/sfc_data.tile4.nc \
+                   RESTART/sfc_data.tile5.nc \
+                   RESTART/sfc_data.tile6.nc"
+
+export_fv3
+export NPZ=127
+export NPZP=128
+export DT_ATMOS=300
+export SYEAR=2021
+export SMONTH=03
+export SDAY=22
+export SHOUR=06
+export RESTART_INTERVAL="6 -1"
+export OUTPUT_GRID='gaussian_grid'
+export NSTF_NAME='2,0,0,0,0'
+export WRITE_DOPOST=.true.
+export IAER=5111
+export OUTPUT_FH='0 09 12'
+export FHMAX=12
+
+export THRD=2
+export TASKS=$TASKS_thrd
+export INPES=$INPES_thrd
+export JNPES=$JNPES_thrd
+export WRTTASK_PER_GROUP=6
+
+export FV3_RUN=control_run.IN
+export CCPP_SUITE=FV3_HRRR
+export INPUT_NML=rap.nml.IN
+export FIELD_TABLE=field_table_thompson_aero_tke
+export NEW_DIAGTABLE=diag_table_rap
+
+export SFCLAY_COMPUTE_FLUX=.true.
+
+export LKM=1
+export IMP_PHYSICS=8
+export DNATS=0
+export DO_SAT_ADJ=.false.
+export LRADAR=.true.
+export LTAEROSOL=.true.
+export IALB=2
+export IEMS=2
+export HYBEDMF=.false.
+export DO_MYNNEDMF=.true.
+export DO_MYNNSFCLAY=.true.
+export DO_DEEP=.false.
+export SHAL_CNV=.false.
+export IMFSHALCNV=-1
+export IMFDEEPCNV=-1
+export LHEATSTRG=.false.
+export LSM=3
+export LSOIL_LSM=9
+export KICE=9
+
+export GWD_OPT=3
+export DO_UGWP_V0=.false.
+export DO_UGWP_V0_OROG_ONLY=.false.
+export DO_GSL_DRAG_LS_BL=.true.
+export DO_GSL_DRAG_SS=.true.
+export DO_GSL_DRAG_TOFD=.true.
+export DO_UGWP_V1=.false.
+export DO_UGWP_V1_OROG_ONLY=.false.
+
diff --git a/tests/tests/hrrr_control_debug_dyn32_phy32 b/tests/tests/hrrr_control_debug_dyn32_phy32
@@ -0,0 +1,66 @@
+##############################################################################
+#
+#  HRRR debug control test: atmosphere only at C96L127
+#
+###############################################################################
+
+export TEST_DESCR="Compare HRRR debug results with previous trunk version"
+
+export CNTL_DIR=hrrr_control_debug_dyn32_phy32
+
+export LIST_FILES="sfcf000.nc \
+                   sfcf001.nc \
+                   atmf000.nc \
+                   atmf001.nc"
+
+export_fv3
+export NPZ=127
+export NPZP=128
+export DT_ATMOS=300
+export SYEAR=2021
+export SMONTH=03
+export SDAY=22
+export SHOUR=06
+export OUTPUT_GRID='gaussian_grid'
+export NSTF_NAME='2,0,0,0,0'
+export IAER=5111
+export OUTPUT_FH='0 1'
+export FHMAX=1
+
+export FV3_RUN=control_run.IN
+export CCPP_SUITE=FV3_HRRR
+export INPUT_NML=rap.nml.IN
+export FIELD_TABLE=field_table_thompson_aero_tke
+export NEW_DIAGTABLE=diag_table_rap
+
+export SFCLAY_COMPUTE_FLUX=.true.
+
+export LKM=1
+export IMP_PHYSICS=8
+export DNATS=0
+export DO_SAT_ADJ=.false.
+export LRADAR=.true.
+export LTAEROSOL=.true.
+export IALB=2
+export IEMS=2
+export HYBEDMF=.false.
+export DO_MYNNEDMF=.true.
+export DO_MYNNSFCLAY=.true.
+export DO_DEEP=.false.
+export SHAL_CNV=.false.
+export IMFSHALCNV=-1
+export IMFDEEPCNV=-1
+export LHEATSTRG=.false.
+export LSM=3
+export LSOIL_LSM=9
+export KICE=9
+
+export GWD_OPT=3
+export DO_UGWP_V0=.false.
+export DO_UGWP_V0_OROG_ONLY=.false.
+export DO_GSL_DRAG_LS_BL=.true.
+export DO_GSL_DRAG_SS=.true.
+export DO_GSL_DRAG_TOFD=.true.
+export DO_UGWP_V1=.false.
+export DO_UGWP_V1_OROG_ONLY=.false.
+
diff --git a/tests/tests/hrrr_control_decomp_dyn32_phy32 b/tests/tests/hrrr_control_decomp_dyn32_phy32
@@ -0,0 +1,111 @@
+###############################################################################
+#
+#  HRRR control test with different PEs
+#
+###############################################################################
+
+export TEST_DESCR="Compare HRRR decomp results with control"
+
+export CNTL_DIR=hrrr_control_dyn32_phy32
+
+export LIST_FILES="sfcf000.nc \
+                   sfcf009.nc \
+                   sfcf012.nc \
+                   atmf000.nc \
+                   atmf009.nc \
+                   atmf012.nc \
+                   GFSFLX.GrbF00 \
+                   GFSFLX.GrbF09 \
+                   GFSFLX.GrbF12 \
+                   GFSPRS.GrbF00 \
+                   GFSPRS.GrbF09 \
+                   GFSPRS.GrbF12 \
+                   RESTART/coupler.res \
+                   RESTART/fv_core.res.nc \
+                   RESTART/fv_core.res.tile1.nc \
+                   RESTART/fv_core.res.tile2.nc \
+                   RESTART/fv_core.res.tile3.nc \
+                   RESTART/fv_core.res.tile4.nc \
+                   RESTART/fv_core.res.tile5.nc \
+                   RESTART/fv_core.res.tile6.nc \
+                   RESTART/fv_srf_wnd.res.tile1.nc \
+                   RESTART/fv_srf_wnd.res.tile2.nc \
+                   RESTART/fv_srf_wnd.res.tile3.nc \
+                   RESTART/fv_srf_wnd.res.tile4.nc \
+                   RESTART/fv_srf_wnd.res.tile5.nc \
+                   RESTART/fv_srf_wnd.res.tile6.nc \
+                   RESTART/fv_tracer.res.tile1.nc \
+                   RESTART/fv_tracer.res.tile2.nc \
+                   RESTART/fv_tracer.res.tile3.nc \
+                   RESTART/fv_tracer.res.tile4.nc \
+                   RESTART/fv_tracer.res.tile5.nc \
+                   RESTART/fv_tracer.res.tile6.nc \
+                   RESTART/phy_data.tile1.nc \
+                   RESTART/phy_data.tile2.nc \
+                   RESTART/phy_data.tile3.nc \
+                   RESTART/phy_data.tile4.nc \
+                   RESTART/phy_data.tile5.nc \
+                   RESTART/phy_data.tile6.nc \
+                   RESTART/sfc_data.tile1.nc \
+                   RESTART/sfc_data.tile2.nc \
+                   RESTART/sfc_data.tile3.nc \
+                   RESTART/sfc_data.tile4.nc \
+                   RESTART/sfc_data.tile5.nc \
+                   RESTART/sfc_data.tile6.nc"
+
+export_fv3
+export NPZ=127
+export NPZP=128
+export DT_ATMOS=300
+export SYEAR=2021
+export SMONTH=03
+export SDAY=22
+export SHOUR=06
+export RESTART_INTERVAL="6 -1"
+export OUTPUT_GRID='gaussian_grid'
+export NSTF_NAME='2,0,0,0,0'
+export WRITE_DOPOST=.true.
+export IAER=5111
+export OUTPUT_FH='0 09 12'
+export FHMAX=12
+
+export INPES=6
+export JNPES=4
+
+export FV3_RUN=control_run.IN
+export CCPP_SUITE=FV3_HRRR
+export INPUT_NML=rap.nml.IN
+export FIELD_TABLE=field_table_thompson_aero_tke
+export NEW_DIAGTABLE=diag_table_rap
+
+export SFCLAY_COMPUTE_FLUX=.true.
+
+export LKM=1
+export IMP_PHYSICS=8
+export DNATS=0
+export DO_SAT_ADJ=.false.
+export LRADAR=.true.
+export LTAEROSOL=.true.
+export IALB=2
+export IEMS=2
+export HYBEDMF=.false.
+export DO_MYNNEDMF=.true.
+export DO_MYNNSFCLAY=.true.
+export DO_DEEP=.false.
+export SHAL_CNV=.false.
+export IMFSHALCNV=-1
+export IMFDEEPCNV=-1
+export LHEATSTRG=.false.
+export LSM=3
+export LSOIL_LSM=9
+export KICE=9
+
+export GWD_OPT=3
+export DO_UGWP_V0=.false.
+export DO_UGWP_V0_OROG_ONLY=.false.
+export DO_GSL_DRAG_LS_BL=.true.
+export DO_GSL_DRAG_SS=.true.
+export DO_GSL_DRAG_TOFD=.true.
+export DO_UGWP_V1=.false.
+export DO_UGWP_V1_OROG_ONLY=.false.
+