Merge pull request #186 from lgannoaa/feature/ccpp

Initial support for CCPP.

driver/product/run_postsnd.sh.dell

@@ -51,7 +51,6 @@ export MP_STDOUTMODE=ordered
 
 #export machine="WCOSS_C"
 machine="WCOSS_DELL_P3"
-#machine="THEIA"
 #machine="JET"
 export npe_postsnd=40
 export npe_postsndcfp=9

jobs/rocoto/getic.sh

@@ -81,7 +81,11 @@ if [ $ics_from = "opsgfs" ]; then
         fanal[1]="./${CDUMP}.$yyyy$mm$dd/$cyc/${CDUMP}.t${cyc}z.atmanl.nemsio"
         fanal[2]="./${CDUMP}.$yyyy$mm$dd/$cyc/${CDUMP}.t${cyc}z.sfcanl.nemsio"
         flanal="${fanal[1]} ${fanal[2]}"
-        tarpref="gpfs_dell1_nco_ops_com"
+        if [ $CDATE -ge "2020022600" ]; then 
+          tarpref="com"
+        else 
+          tarpref="gpfs_dell1_nco_ops_com"
+        fi
         if [ $CDUMP = "gdas" ]; then
             tarball="$hpssdir/${tarpref}_gfs_prod_${CDUMP}.${yyyy}${mm}${dd}_${cyc}.${CDUMP}_nemsio.tar"
         elif [ $CDUMP = "gfs" ]; then

parm/config/config.base.emc.dyn

@@ -139,6 +139,10 @@ export LEVS=128
 export CASE="@CASECTL@"
 export CASE_ENKF="@CASEENS@"
 
+# Run with CCPP physics
+export RUN_CCPP="NO"
+export CCPP_SUITE="FV3_GFS_v16beta"
+
 # Surface cycle update frequency
 if [[ "$CDUMP" == "gdas" ]] ; then
    export FHCYC=1

parm/config/config.base.nco.static

@@ -126,6 +126,10 @@ export LEVS=128
 export CASE="C768"
 export CASE_ENKF="C384"
 
+# Run with CCPP physics
+export RUN_CCPP="NO"
+export CCPP_SUITE="FV3_GFS_v16beta"
+
 # Surface cycle update frequency
 if [[ "$CDUMP" == "gdas" ]] ; then
    export FHCYC=1

parm/config/config.fcst

@@ -89,8 +89,9 @@ fi
 
 # Radiation options 
 export IAER=5111    ;#spectral band mapping method for aerosol optical properties
-export iovr_lw=3    ;#de-correlation length cloud overlap method (Barker, 2008)
-export iovr_sw=3    ;#de-correlation length cloud overlap method (Barker, 2008)
+export iovr_lw=3    ;#de-correlation length cloud overlap method (Barker, 2008) 
+export iovr_sw=3    ;#de-correlation length cloud overlap method (Barker, 2008) 
+export iovr=3       ;#de-correlation length cloud overlap method (Barker, 2008) 
 export icliq_sw=2   ;#cloud optical coeffs from AER's newer version v3.9-v4.0 for hu and stamnes
 
 

parm/config/config.gldas

@@ -12,5 +12,4 @@ export GLDASSH=$HOMEgfs/scripts/exgdas_atmos_gldas.sh
 export gldas_spinup_hours=72
 export CPCGAUGE=$DMPDIR
 
-
 echo "END: config.gldas"

parm/config/config.prep

@@ -17,10 +17,7 @@ export DO_RELOCATE="NO"
 export TROPCYQCRELOSH="$HOMEgfs/scripts/exglobal_atmos_tropcy_qc_reloc.sh"
 export SENDCOM=YES
 
-export COMINsyn=${COMINsyn:-$(compath.py gfs/prod/syndat)}
-if [ $machine = "HERA" ]; then
-    export COMINsyn="${COMROOT}/arch/prod/syndat"
-fi
+export COMINsyn=${COMINsyn:-${COMROOT}/gfs/prod/syndat}
 
 export HOMERELO=$HOMEgfs
 export EXECRELO=${HOMERELO}/exec

parm/config/config.resources

@@ -301,7 +301,7 @@ elif [ $step = "eupd" ]; then
 
     export wtime_eupd="00:30:00"
     if [ $CASE = "C768" ]; then
-      export npe_eupd=960
+      export npe_eupd=480
       export nth_eupd=6
       if [[ "$machine" = "WCOSS_DELL_P3" ]]; then
         export nth_eupd=7
@@ -355,7 +355,11 @@ elif [ $step = "esfc" ]; then
 
 elif [ $step = "efcs" ]; then
 
-    export wtime_efcs="00:40:00"
+    if [ $CASE = "C768" ]; then
+      export wtime_efcs="01:00:00"
+    else
+      export wtime_efcs="00:40:00"
+    fi
     export npe_efcs=$(echo "$layout_x * $layout_y * 6" | bc)
     export nth_efcs=${nth_fv3:-2}
     export npe_node_efcs=$(echo "$npe_node_max / $nth_efcs" | bc)

parm/config/config.vrfy

@@ -159,11 +159,6 @@ if [ "$VRFYTRAK" = "YES" ]; then
         export FHOUT_CYCLONE=6            
         export FHMAX_CYCLONE=$(( FHMAX_GFS<240 ? FHMAX_GFS : 240 ))
     fi
-    if [ $machine = "HERA" ]; then
-       export COMROOT="/scratch1/NCEPDEV/global/glopara/com"
-    elif [ $machine = "ORION" ]; then
-       export COMROOT="/work/noaa/global/glopara/com"
-    fi
     export COMINsyn=${COMINsyn:-${COMROOT}/gfs/prod/syndat}
 fi
 

parm/parm_fv3diag/field_table_gsd

@@ -0,0 +1,65 @@
+# added by FRE: sphum must be present in atmos
+# specific humidity for moist runs
+ "TRACER", "atmos_mod", "sphum"
+           "longname",     "specific humidity"
+           "units",        "kg/kg"
+       "profile_type", "fixed", "surface_value=3.e-6" /
+# prognostic cloud water mixing ratio
+ "TRACER", "atmos_mod", "liq_wat"
+           "longname",     "cloud water mixing ratio"
+           "units",        "kg/kg"
+       "profile_type", "fixed", "surface_value=1.e30" /
+# prognostic ice water mixing ratio
+ "TRACER", "atmos_mod", "ice_wat"
+           "longname",     "cloud ice mixing ratio"
+           "units",        "kg/kg"
+       "profile_type", "fixed", "surface_value=1.e30" /
+# prognostic rain water mixing ratio
+ "TRACER", "atmos_mod", "rainwat"
+           "longname",     "rain water mixing ratio"
+           "units",        "kg/kg"
+       "profile_type", "fixed", "surface_value=1.e30" /
+# prognostic snow water mixing ratio
+ "TRACER", "atmos_mod", "snowwat"
+           "longname",     "snow water mixing ratio"
+           "units",        "kg/kg"
+       "profile_type", "fixed", "surface_value=1.e30" /
+# prognostic Grau water mixing ratio
+ "TRACER", "atmos_mod", "graupel"
+           "longname",     "graupel mixing ratio"
+           "units",        "kg/kg"
+       "profile_type", "fixed", "surface_value=1.e30" /
+# prognostic cloud water number concentration
+ "TRACER", "atmos_mod", "water_nc"
+           "longname",     "cloud  liquid water  number concentration"
+           "units",        "/kg"
+       "profile_type", "fixed", "surface_value=1.e30" /
+# prognostic cloud ice number concentration
+ "TRACER", "atmos_mod", "ice_nc"
+           "longname",     "cloud ice water number concentration"
+           "units",        "/kg"
+       "profile_type", "fixed", "surface_value=1.e30" /
+# prognostic rain number concentration
+ "TRACER", "atmos_mod", "rain_nc"
+           "longname",     "rain number concentration"
+           "units",        "/kg"
+       "profile_type", "fixed", "surface_value=1.e30" /
+# prognostic ozone mixing ratio tracer
+ "TRACER", "atmos_mod", "o3mr"
+           "longname",     "ozone mixing ratio"
+           "units",        "kg/kg"
+       "profile_type", "fixed", "surface_value=1.e30" /
+# water- and ice-friendly aerosols (Thompson)
+ "TRACER", "atmos_mod", "liq_aero"
+           "longname",     "water-friendly aerosol number concentration"
+           "units",        "/kg"
+       "profile_type", "fixed", "surface_value=0.0" /
+ "TRACER", "atmos_mod", "ice_aero"
+           "longname",     "ice-friendly aerosol number concentration"
+           "units",        "/kg"
+       "profile_type", "fixed", "surface_value=0.0" /
+# prognostic subgrid scale turbulent kinetic energy
+ "TRACER", "atmos_mod", "sgs_tke"
+           "longname",     "subgrid scale turbulent kinetic energy"
+           "units",        "m2/s2"
+       "profile_type", "fixed", "surface_value=1.e30" /

scripts/exglobal_forecast.sh

@@ -116,6 +116,7 @@ NTASKS_FV3=${NTASKS_FV3:-$npe_fv3}
 
 TYPE=${TYPE:-"nh"}                  # choices:  nh, hydro
 MONO=${MONO:-"non-mono"}            # choices:  mono, non-mono
+RUN_CCPP=${RUN_CCPP:-"NO"}
 
 QUILTING=${QUILTING:-".true."}
 OUTPUT_GRID=${OUTPUT_GRID:-"gaussian_grid"}
@@ -726,6 +727,16 @@ fi
 $NCP $DATA_TABLE  data_table
 $NCP $FIELD_TABLE field_table
 
+# copy CCN_ACTIVATE.BIN for Thompson microphysics
+if [ $RUN_CCPP = "YES" ]; then
+if [ "$CCPP_SUITE" = 'FV3_GSD_v0' -o "$CCPP_SUITE" = 'FV3_GSD_noah' ]; then 
+  $NLN $FIX_AM/CCN_ACTIVATE.BIN  CCN_ACTIVATE.BIN
+  $NLN $FIX_AM/freezeH2O.dat  freezeH2O.dat
+  $NLN $FIX_AM/qr_acr_qg.dat  qr_acr_qg.dat
+  $NLN $FIX_AM/qr_acr_qs.dat  qr_acr_qs.dat
+fi
+fi
+
 #------------------------------------------------------------------
 rm -f nems.configure
 
@@ -859,6 +870,11 @@ EOF
 #  $coupler_nml
 #/
 
+atmos_model_nml=""
+if [ $RUN_CCPP = "YES" ]; then
+ atmos_model_nml="ccpp_suite = $CCPP_SUITE"
+fi
+
 cat > input.nml <<EOF
 &amip_interp_nml
   interp_oi_sst = .true.
@@ -1019,8 +1035,6 @@ deflate_level=${deflate_level:-1}
   iems         = ${IEMS:-"1"}
   iaer         = $IAER
   icliq_sw     = ${icliq_sw:-"2"}
-  iovr_lw      = ${iovr_lw:-"3"}
-  iovr_sw      = ${iovr_sw:-"3"}
   ico2         = $ICO2
   isubc_sw     = ${isubc_sw:-"2"}
   isubc_lw     = ${isubc_lw:-"2"}
@@ -1075,6 +1089,31 @@ deflate_level=${deflate_level:-1}
   do_skeb      = ${do_skeb:-".false."}
 EOF
 
+if [ $RUN_CCPP = "YES" ]; then
+  cat >> input.nml << EOF
+  iovr         = ${iovr:-"3"}
+  ltaerosol    = ${ltaerosol:-".false."}
+  lradar       = ${lradar:-".false."}
+  ttendlim     = ${ttendlim:-"0.005"}
+  oz_phys      = ${oz_phys:-".false."}
+  oz_phys_2015 = ${oz_phys_2015:-".true."}
+  lsoil_lsm    = ${lsoil_lsm:-"4"}
+  do_mynnedmf  = ${do_mynnedmf:-".false."}
+  do_mynnsfclay = ${do_mynnsfclay:-".false."}
+  icloud_bl    = ${icloud_bl:-"1"}
+  bl_mynn_edmf = ${bl_mynn_edmf:-"1"}
+  bl_mynn_tkeadvect = ${bl_mynn_tkeadvect:-".true."}
+  bl_mynn_edmf_mom = ${bl_mynn_edmf_mom:-"1"}
+  min_lakeice  = ${min_lakeice:-"0.15"}
+  min_seaice   = ${min_seaice:-"0.15"}
+EOF
+else
+  cat >> input.nml << EOF
+  iovr_lw      = ${iovr_lw:-"3"}
+  iovr_sw      = ${iovr_sw:-"3"}
+EOF
+fi
+
 # Add namelist for IAU
 if [ $DOIAU = "YES" ]; then
   cat >> input.nml << EOF
@@ -1306,6 +1345,9 @@ else
   done
 fi
 
+# Copy namelist file
+$NCP input.nml $memdir
+
 #------------------------------------------------------------------
 # run the executable
 

sorc/build_all.sh

@@ -9,6 +9,18 @@ set +x
 #                   Anything other than "true"  will use libraries locally.
 #------------------------------------
 
+while getopts "c" option;
+do
+ case $option in
+  c)
+   echo "Received -c flag, check out ufs-weather-model develop branch with CCPP physics"
+   RUN_CCPP="YES"
+   ;;
+ esac
+done
+
+
+
 export USE_PREINST_LIBS="true"
 
 #------------------------------------
@@ -60,6 +72,7 @@ echo " .... Library build not currently supported .... "
 #------------------------------------
 $Build_fv3gfs && {
 echo " .... Building fv3 .... "
+export RUN_CCPP=${RUN_CCPP:-"NO"}
 ./build_fv3.sh > $logs_dir/build_fv3.log 2>&1
 rc=$?
 if [[ $rc -ne 0 ]] ; then

sorc/build_fv3.sh

@@ -22,5 +22,11 @@ if [ $target = orion ]; then target=orion.intel ; fi
 cd fv3gfs.fd/
 FV3=$( pwd -P )/FV3
 cd tests/
-./compile.sh "$FV3" "$target" "WW3=Y 32BIT=Y" 1
-mv -f fv3_1.exe ../NEMS/exe/global_fv3gfs.x
+
+if [ ${RUN_CCPP:-${1:-"NO"}} = "NO" ]; then
+ ./compile.sh "$FV3" "$target" "WW3=Y 32BIT=Y" 1
+ mv -f fv3_1.exe ../NEMS/exe/global_fv3gfs.x
+else
+ ./compile.sh "$target" "CCPP=Y 32BIT=Y SUITES=FV3_GFS_v15,FV3_GFS_v16beta" 2 NO NO
+ mv -f fv3_2.exe ../NEMS/exe/global_fv3gfs.x
+fi

sorc/checkout.sh

@@ -1,13 +1,18 @@
 #!/bin/sh
-set -xue
+#set -xue
+set -x
 
-while getopts "o" option;
+while getopts "oc" option;
 do
  case $option in
   o)
    echo "Received -o flag for optional checkout of GTG, will check out GTG with EMC_post"
    checkout_gtg="YES"
    ;;
+  c)
+   echo "Received -c flag, check out ufs-weather-model develop branch with CCPP physics"  
+   run_ccpp="YES"
+   ;;
   :)
    echo "option -$OPTARG needs an argument"
    ;;
@@ -26,7 +31,11 @@ if [[ ! -d fv3gfs.fd ]] ; then
     rm -f ${topdir}/checkout-fv3gfs.log
     git clone https://github.com/ufs-community/ufs-weather-model fv3gfs.fd >> ${topdir}/checkout-fv3gfs.log 2>&1
     cd fv3gfs.fd
-    git checkout GFS.v16.0.14
+    if [ ${run_ccpp:-"NO"} = "NO" ]; then
+     git checkout GFS.v16.0.14
+    else
+     git checkout 2e25df5fe952d27355ed58963148f46b82565469
+    fi
     git submodule update --init --recursive
     cd ${topdir}
 else

sorc/partial_build.sh

@@ -152,8 +152,10 @@
        echo "Usage: $0 [ALL|config=config_file|[select=][prog1[,prog2[,...]]]" 2>&1
        exit 2
      }
-     ( [[ $1 == "-v" ]] || [[ ${1,,} == "--verbose" ]] ) && {
-       verbose=true
+     ( [[ $1 == "-v" ]] || [[ ${1,,} == "--verbose" ]] || [[ $1 == "-c" ]] ) && {
+       if [[ $1 == "-v" ]]; then
+         verbose=true
+       fi
        num_arg=0
      } || {
        echo "Usage: $0 [ALL|config=config_file|[select=][prog1[,prog2[,...]]]" 2>&1

ush/rocoto/setup_expt.py

@@ -240,6 +240,7 @@ def edit_baseconfig():
       base_svn = '/work/noaa/global/glopara/svn'
       dmpdir = '/work/noaa/global/glopara/dump'
       nwprod = '/work/noaa/global/glopara/nwpara'
+      comroot = '/work/noaa/global/glopara/com'
       homedir = '/work/noaa/global/$USER'
       stmp = '/work/noaa/stmp/$USER'
       ptmp = '/work/noaa/stmp/$USER'

ush/rocoto/setup_expt_fcstonly.py

@@ -206,6 +206,7 @@ def edit_baseconfig():
       base_svn = '/work/noaa/global/glopara/svn'
       dmpdir = '/work/noaa/global/glopara/dump'
       nwprod = '/work/noaa/global/glopara/nwpara'
+      comroot = '/work/noaa/global/glopara/com'
       homedir = '/work/noaa/global/$USER'
       stmp = '/work/noaa/stmp/$USER'
       ptmp = '/work/noaa/stmp/$USER'