Merge remote-tracking branch 'emc/develop' into ss160

ci/cases/pr/C48C48_ufs_hybatmDA.yaml

@@ -0,0 +1,22 @@
+experiment:
+  system: gfs
+  mode: cycled
+
+arguments:
+  pslot: {{ 'pslot' | getenv }}
+  app: ATM
+  resdetatmos: 48
+  resensatmos: 48
+  comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
+  expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
+  idate: 2021032312
+  edate: 2021032400
+  nens: 2
+  gfs_cyc: 1
+  start: warm
+  yaml: {{ HOMEgfs }}/ci/cases/yamls/ufs_hybatmDA_defaults.ci.yaml
+
+skip_ci_on_hosts:
+  - hera
+  - orion
+  - hercules

ci/cases/pr/C48_S2SWA_gefs.yaml

@@ -16,3 +16,8 @@ arguments:
   idate: 2021032312
   edate: 2021032312
   yaml: {{ HOMEgfs }}/ci/cases/yamls/gefs_ci_defaults.yaml
+
+skip_ci_on_hosts:
+  - hera
+  - orion
+  - hercules

ci/cases/pr/C48mx500_3DVarAOWCDA.yaml

@@ -17,7 +17,7 @@ arguments:
   start: warm
   yaml: {{ HOMEgfs }}/ci/cases/yamls/soca_gfs_defaults_ci.yaml
 
-skip_ci_on_host:
+skip_ci_on_hosts:
   - orion
   - hera
   - hercules

ci/cases/yamls/build.yaml

@@ -1,3 +1,3 @@
 builds:
  - gefs: './build_all.sh'
- - gfs: './build_all.sh -gu'
+ - gfs: './build_all.sh -wgu'

ci/cases/yamls/ufs_hybatmDA_defaults.ci.yaml

@@ -0,0 +1,14 @@
+defaults:
+  !INC {{ HOMEgfs }}/parm/config/gfs/yaml/defaults.yaml
+base:
+  DOIAU: "NO"
+  DO_JEDIATMVAR: "YES"
+  DO_JEDIATMENS: "YES"
+  ACCOUNT: {{ 'SLURM_ACCOUNT' | getenv }}
+esfc:
+  DONST: "NO"
+nsst:
+  NST_MODEL: "1"
+sfcanl:
+  DONST: "NO"
+

ci/scripts/check_ci.sh

@@ -8,7 +8,7 @@ set -eux
 #                     to run from within a cron job in the CI Managers account
 #####################################################################################
 
-ROOT_DIR="$(cd "$(dirname  "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd )"
+HOMEgfs="$(cd "$(dirname  "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd )"
 scriptname=$(basename "${BASH_SOURCE[0]}")
 echo "Begin ${scriptname} at $(date -u)" || true
 export PS4='+ $(basename ${BASH_SOURCE})[${LINENO}]'
@@ -20,21 +20,22 @@ REPO_URL="https://github.com/NOAA-EMC/global-workflow.git"
 #  Set up runtime environment varibles for accounts on supproted machines
 #########################################################################
 
-source "${ROOT_DIR}/ush/detect_machine.sh"
+source "${HOMEgfs}/ush/detect_machine.sh"
 case ${MACHINE_ID} in
   hera | orion | hercules)
    echo "Running Automated Testing on ${MACHINE_ID}"
-   source "${ROOT_DIR}/ci/platforms/config.${MACHINE_ID}"
+   source "${HOMEgfs}/ci/platforms/config.${MACHINE_ID}"
    ;;
  *)
    echo "Unsupported platform. Exiting with error."
    exit 1
    ;;
 esac
 set +x
-source "${ROOT_DIR}/ush/module-setup.sh"
-source "${ROOT_DIR}/ci/scripts/utils/ci_utils.sh"
-module use "${ROOT_DIR}/modulefiles"
+export HOMEgfs
+source "${HOMEgfs}/ush/module-setup.sh"
+source "${HOMEgfs}/ci/scripts/utils/ci_utils.sh"
+module use "${HOMEgfs}/modulefiles"
 module load "module_gwsetup.${MACHINE_ID}"
 module list
 set -x
@@ -57,7 +58,7 @@ pr_list_dbfile="${GFS_CI_ROOT}/open_pr_list.db"
 
 pr_list=""
 if [[ -f "${pr_list_dbfile}" ]]; then
-  pr_list=$("${ROOT_DIR}/ci/scripts/pr_list_database.py" --dbfile "${pr_list_dbfile}" --display | grep -v Failed | grep Running | awk '{print $1}') || true
+  pr_list=$("${HOMEgfs}/ci/scripts/pr_list_database.py" --dbfile "${pr_list_dbfile}" --display | grep -v Failed | grep Running | awk '{print $1}') || true
 fi
 if [[ -z "${pr_list+x}" ]]; then
   echo "no PRs open and ready to run cases on .. exiting"
@@ -89,13 +90,13 @@ for pr in ${pr_list}; do
     sed -i "1 i\`\`\`" "${output_ci}"
     sed -i "1 i\All CI Test Cases Passed on ${MACHINE_ID^}:" "${output_ci}"
     "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${output_ci}"
-    "${ROOT_DIR}/ci/scripts/pr_list_database.py" --remove_pr "${pr}" --dbfile "${pr_list_dbfile}"
+    "${HOMEgfs}/ci/scripts/pr_list_database.py" --remove_pr "${pr}" --dbfile "${pr_list_dbfile}"
     # Check to see if this PR that was opened by the weekly tests and if so close it if it passed on all platforms
     weekly_labels=$(${GH} pr view "${pr}" --repo "${REPO_URL}"  --json headRefName,labels,author --jq 'select(.author.login | contains("emcbot")) | select(.headRefName | contains("weekly_ci")) | .labels[].name ') || true
     if [[ -n "${weekly_labels}" ]]; then
-      num_platforms=$(find "${ROOT_DIR}/ci/platforms" -type f -name "config.*" | wc -l)
+      num_platforms=$(find "${HOMEgfs}/ci/platforms" -type f -name "config.*" | wc -l)
       passed=0
-      for platforms in "${ROOT_DIR}"/ci/platforms/config.*; do
+      for platforms in "${HOMEgfs}"/ci/platforms/config.*; do
         machine=$(basename "${platforms}" | cut -d. -f2)
         if [[ "${weekly_labels}" == *"CI-${machine^}-Passed"* ]]; then
           ((passed=passed+1))
@@ -139,7 +140,7 @@ for pr in ${pr_list}; do
       } >> "${output_ci}"
       sed -i "1 i\`\`\`" "${output_ci}"
       "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${output_ci}"
-      "${ROOT_DIR}/ci/scripts/pr_list_database.py" --remove_pr "${pr}" --dbfile "${pr_list_dbfile}"
+      "${HOMEgfs}/ci/scripts/pr_list_database.py" --remove_pr "${pr}" --dbfile "${pr_list_dbfile}"
       for kill_cases in "${pr_dir}/RUNTESTS/"*; do
          pslot=$(basename "${kill_cases}")
          cancel_slurm_jobs "${pslot}"

ci/scripts/clone-build_ci.sh

@@ -74,7 +74,7 @@ set +e
 source "${HOMEgfs}/ush/module-setup.sh"
 export BUILD_JOBS=8
 rm -rf log.build
-./build_all.sh -gu  >> log.build 2>&1
+./build_all.sh -guw  >> log.build 2>&1
 build_status=$?
 
 DATE=$(date +'%D %r')

ci/scripts/driver.sh

@@ -47,12 +47,15 @@ esac
 ######################################################
 # setup runtime env for correct python install and git
 ######################################################
+HOMEgfs=${ROOT_DIR}
+export HOMEgfs
 set +x
 source "${ROOT_DIR}/ci/scripts/utils/ci_utils.sh"
 source "${ROOT_DIR}/ush/module-setup.sh"
 module use "${ROOT_DIR}/modulefiles"
 module load "module_gwsetup.${MACHINE_ID}"
 set -x
+unset HOMEgfs
 
 ############################################################
 # query repo and get list of open PRs with tags {machine}-CI

ci/scripts/run_ci.sh

@@ -9,7 +9,7 @@ set -eux
 # Abstract TODO
 #####################################################################################
 
-ROOT_DIR="$(cd "$(dirname  "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd )"
+HOMEgfs="$(cd "$(dirname  "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd )"
 scriptname=$(basename "${BASH_SOURCE[0]}")
 echo "Begin ${scriptname} at $(date -u)" || true
 export PS4='+ $(basename ${BASH_SOURCE})[${LINENO}]'
@@ -18,20 +18,21 @@ export PS4='+ $(basename ${BASH_SOURCE})[${LINENO}]'
 #  Set up runtime environment varibles for accounts on supproted machines
 #########################################################################
 
-source "${ROOT_DIR}/ush/detect_machine.sh"
+source "${HOMEgfs}/ush/detect_machine.sh"
 case ${MACHINE_ID} in
   hera | orion | hercules)
    echo "Running Automated Testing on ${MACHINE_ID}"
-   source "${ROOT_DIR}/ci/platforms/config.${MACHINE_ID}"
+   source "${HOMEgfs}/ci/platforms/config.${MACHINE_ID}"
    ;;
  *)
    echo "Unsupported platform. Exiting with error."
    exit 1
    ;;
 esac
 set +x
-source "${ROOT_DIR}/ush/module-setup.sh"
-module use "${ROOT_DIR}/modulefiles"
+export HOMEgfs
+source "${HOMEgfs}/ush/module-setup.sh"
+module use "${HOMEgfs}/modulefiles"
 module load "module_gwsetup.${MACHINE_ID}"
 module list
 set -eux
@@ -47,7 +48,7 @@ pr_list_dbfile="${GFS_CI_ROOT}/open_pr_list.db"
 
 pr_list=""
 if [[ -f "${pr_list_dbfile}" ]]; then
-  pr_list=$("${ROOT_DIR}/ci/scripts/pr_list_database.py" --display --dbfile "${pr_list_dbfile}" | grep -v Failed | grep Open | grep Running | awk '{print $1}' | head -"${max_concurrent_pr}") || true
+  pr_list=$("${HOMEgfs}/ci/scripts/pr_list_database.py" --display --dbfile "${pr_list_dbfile}" | grep -v Failed | grep Open | grep Running | awk '{print $1}' | head -"${max_concurrent_pr}") || true
 fi
 if [[ -z "${pr_list}" ]]; then
   echo "no open and built PRs that are ready for the cases to advance with rocotorun .. exiting"

docs/source/clone.rst

@@ -39,6 +39,13 @@ For coupled cycling (include new UFSDA) use the `-gu` options during build:
    ./build_all.sh -gu
 
 
+For building with PDLIB for the wave model, use the `-w` options during build:
+
+::
+
+   ./build_all.sh -w
+
+
 Build workflow components and link workflow artifacts such as executables, etc.
 
 ::

env/WCOSS2.env

@@ -176,6 +176,8 @@ elif [[ "${step}" = "fcst" ]] || [[ "${step}" = "efcs" ]]; then
     if [[ "${step}" = "fcst" ]]; then
         export OMP_PLACES=cores
         export OMP_STACKSIZE=2048M
+        export MPICH_MPIIO_HINTS="*:romio_cb_write=disable"
+        export FI_OFI_RXM_SAR_LIMIT=3145728
     elif [[ "${step}" = "efcs" ]]; then
         export MPICH_MPIIO_HINTS="*:romio_cb_write=disable"
         export FI_OFI_RXM_SAR_LIMIT=3145728

parm/config/gfs/config.base.emc.dyn

@@ -39,6 +39,7 @@ export FIXcice="${FIXgfs}/cice"
 export FIXmom="${FIXgfs}/mom6"
 export FIXreg2grb2="${FIXgfs}/reg2grb2"
 export FIXugwd="${FIXgfs}/ugwd"
+export FIXgdas="${FIXgfs}/gdas"
 
 ########################################################################
 
@@ -182,16 +183,16 @@ export ICERES="${OCNRES}"
 # These are the currently recommended grid-combinations
 case "${CASE}" in
     "C48")
-        export waveGRD='glo_500'
+        export waveGRD='uglo_100km'
         ;;
     "C96" | "C192")
-        export waveGRD='glo_200'
+        export waveGRD='uglo_100km'
         ;;
     "C384")
-        export waveGRD='glo_025'
+        export waveGRD='uglo_100km'
         ;;
     "C768" | "C1152")
-        export waveGRD='mx025'
+        export waveGRD='uglo_m1g16'
         ;;
     *)
         echo "FATAL ERROR: Unrecognized CASE ${CASE}, ABORT!"
@@ -396,4 +397,11 @@ export FITSARC="YES"
 export FHMAX_FITS=132
 [[ "${FHMAX_FITS}" -gt "${FHMAX_GFS}" ]] && export FHMAX_FITS=${FHMAX_GFS}
 
+# The monitor jobs are not yet supported for JEDIATMVAR
+if [[ ${DO_JEDIATMVAR} = "YES" ]]; then
+   export DO_VERFOZN="NO"     # Ozone data assimilation monitoring
+   export DO_VERFRAD="NO"     # Radiance data assimilation monitoring
+   export DO_VMINMON="NO"     # GSI minimization monitoring
+fi
+
 echo "END: config.base"

parm/config/gfs/config.esfc

@@ -16,4 +16,9 @@ if [ $DOIAU_ENKF = "YES" ]; then
    export DOSFCANL_ENKF="NO"
 fi
 
+# Turn off NST in JEDIATMENS
+if [[ "${DO_JEDIATMENS}" == "YES" ]]; then
+   export DONST="NO"
+fi
+
 echo "END: config.esfc"

parm/config/gfs/config.nsst

@@ -10,6 +10,11 @@ echo "BEGIN: config.nsst"
 # nstf_name(1) : NST_MODEL (NSST Model) : 0 = OFF, 1 = ON but uncoupled, 2 = ON and coupled
 export NST_MODEL=2
 
+# Set NST_MODEL for JEDIATMVAR or JEDIATMENS
+if [[ "${DO_JEDIATMVAR}" == "YES" || "${DO_JEDIATMENS}" == "YES" ]]; then
+   export NST_MODEL=1
+fi
+
 # nstf_name(2) : NST_SPINUP : 0 = OFF, 1 = ON,
 export NST_SPINUP=0
 cdate="${PDY}${cyc}"

parm/config/gfs/config.resources

@@ -91,7 +91,7 @@ case ${step} in
     ;;
 
   "prepatmiodaobs")
-    export wtime_prepatmiodaobs="00:10:00"
+    export wtime_prepatmiodaobs="00:30:00"
     export npe_prepatmiodaobs=1
     export nth_prepatmiodaobs=1
     export npe_node_prepatmiodaobs=$(( npe_node_max / nth_prepatmiodaobs ))
@@ -196,8 +196,8 @@ case ${step} in
 
   "atmanlinit")
     # make below case dependent later
-    export layout_x=1
-    export layout_y=1
+    export layout_x=8
+    export layout_y=8
 
     export layout_gsib_x=$(( layout_x * 3 ))
     export layout_gsib_y=$(( layout_y * 2 ))
@@ -212,15 +212,16 @@ case ${step} in
 
   "atmanlrun")
     # make below case dependent later
-    export layout_x=1
-    export layout_y=1
+    export layout_x=8
+    export layout_y=8
 
     export wtime_atmanlrun="00:30:00"
     export npe_atmanlrun=$(( layout_x * layout_y * 6 ))
     export npe_atmanlrun_gfs=$(( layout_x * layout_y * 6 ))
     export nth_atmanlrun=1
     export nth_atmanlrun_gfs=${nth_atmanlrun}
     export npe_node_atmanlrun=$(( npe_node_max / nth_atmanlrun ))
+    export memory_atmanlrun="96GB"
     export is_exclusive=True
     ;;
 
@@ -737,15 +738,15 @@ case ${step} in
     export npe_genesis=1
     export nth_genesis=1
     export npe_node_genesis=1
-    export memory_genesis="4G"
+    export memory_genesis="10G"
     ;;
 
   "genesis_fsu")
     export wtime_genesis_fsu="00:10:00"
     export npe_genesis_fsu=1
     export nth_genesis_fsu=1
     export npe_node_genesis_fsu=1
-    export memory_genesis_fsu="4G"
+    export memory_genesis_fsu="10G"
     ;;
 
   "fit2obs")
@@ -829,8 +830,8 @@ case ${step} in
 
   "atmensanlinit")
     # make below case dependent later
-    export layout_x=1
-    export layout_y=1
+    export layout_x=8
+    export layout_y=8
 
     export wtime_atmensanlinit="00:10:00"
     export npe_atmensanlinit=1
@@ -841,15 +842,16 @@ case ${step} in
 
   "atmensanlrun")
     # make below case dependent later
-    export layout_x=1
-    export layout_y=1
+    export layout_x=8
+    export layout_y=8
 
     export wtime_atmensanlrun="00:30:00"
     export npe_atmensanlrun=$(( layout_x * layout_y * 6 ))
     export npe_atmensanlrun_gfs=$(( layout_x * layout_y * 6 ))
     export nth_atmensanlrun=1
     export nth_atmensanlrun_gfs=${nth_atmensanlrun}
     export npe_node_atmensanlrun=$(( npe_node_max / nth_atmensanlrun ))
+    export memory_atmensanlrun="96GB"
     export is_exclusive=True
     ;;
 

parm/config/gfs/config.sfcanl

@@ -8,4 +8,9 @@ echo "BEGIN: config.sfcanl"
 # Get task specific resources
 . $EXPDIR/config.resources sfcanl
 
+# Turn off NST in JEDIATMVAR
+if [[ "${DO_JEDIATMVAR}" == "YES" ]]; then
+   export DONST="NO"
+fi
+
 echo "END: config.sfcanl"