Add ability to process ocean/ice products specific to GEFS

parm/config/gefs/config.base

@@ -211,6 +211,13 @@ case "${APP}" in
     ;;
 esac
 
+# Output frequency of the forecast model (for cycling)
+export FHMIN=0
+export FHMAX=9
+export FHOUT=3           # Will be changed to 1 in config.base if (DOHYBVAR set to NO and l4densvar set to false)
+export FHOUT_OCN=3
+export FHOUT_ICE=3
+
 # GFS cycle info
 export gfs_cyc=@gfs_cyc@ # 0: no GFS cycle, 1: 00Z only, 2: 00Z and 12Z only, 4: all 4 cycles.
 
@@ -221,7 +228,8 @@ export FHMAX_GFS=@FHMAX_GFS@
 export FHOUT_GFS=6
 export FHMAX_HF_GFS=0
 export FHOUT_HF_GFS=1
-export FHOUT_OCNICE_GFS=6
+export FHOUT_OCN_GFS=6
+export FHOUT_ICE_GFS=6
 if (( gfs_cyc != 0 )); then
     export STEP_GFS=$(( 24 / gfs_cyc ))
 else

parm/config/gefs/config.fcst

@@ -30,7 +30,8 @@ export FHMAX=${FHMAX_GFS}
 export FHOUT=${FHOUT_GFS}
 export FHMAX_HF=${FHMAX_HF_GFS}
 export FHOUT_HF=${FHOUT_HF_GFS}
-export FHOUT_OCNICE=${FHOUT_OCNICE_GFS}
+export FHOUT_OCN=${FHOUT_OCN_GFS}
+export FHOUT_ICE=${FHOUT_ICE_GFS} 
 
 # Get task specific resources
 source "${EXPDIR}/config.resources" fcst

parm/config/gefs/config.oceanice_products

@@ -0,0 +1,15 @@
+#! /usr/bin/env bash
+
+########## config.oceanice_products ##########
+
+echo "BEGIN: config.oceanice_products"
+
+# Get task specific resources
+source "${EXPDIR}/config.resources" oceanice_products
+
+export OCEANICEPRODUCTS_CONFIG="${PARMgfs}/post/oceanice_products_gefs.yaml"
+
+# No. of forecast hours to process in a single job
+export NFHRS_PER_GROUP=3
+
+echo "END: config.oceanice_products"

parm/config/gefs/config.resources

@@ -198,8 +198,8 @@ case ${step} in
         declare -x "wtime_${step}_gfs"="03:00:00"
         ;;
       "C384" | "C768" | "C1152")
-        declare -x "wtime_${step}"="06:00:00"
-        declare -x "wtime_${step}_gfs"="06:00:00"
+        declare -x "wtime_${step}"="14:00:00"
+        declare -x "wtime_${step}_gfs"="14:00:00"
         ;;
       *)
         echo "FATAL ERROR: Resources not defined for job ${job} at resolution ${CASE}"
@@ -281,7 +281,7 @@ case ${step} in
     ;;
 
   "wavepostpnt")
-    export wtime_wavepostpnt="04:00:00"
+    export wtime_wavepostpnt="08:00:00"
     export npe_wavepostpnt=200
     export nth_wavepostpnt=1
     export npe_node_wavepostpnt=$(( npe_node_max / nth_wavepostpnt ))

parm/config/gefs/config.ufs

@@ -143,14 +143,14 @@ case "${fv3_res}" in
         ;;
     "C384")
         export DELTIM=300
-        export layout_x=6
-        export layout_y=8
-        export layout_x_gfs=8
-        export layout_y_gfs=8
+        export layout_x=16
+        export layout_y=16
+        export layout_x_gfs=16
+        export layout_y_gfs=16
         export nthreads_fv3=1
-        export nthreads_fv3_gfs=2
+        export nthreads_fv3_gfs=1
         export nthreads_ufs=1
-        export nthreads_ufs_gfs=2
+        export nthreads_ufs_gfs=1
         export cdmbgwd="1.1,0.72,1.0,1.0"  # mountain blocking, ogwd, cgwd, cgwd src scaling
         export cdmbgwd_gsl="5.0,5.0,1.0,1.0"   # settings for GSL drag suite
         export knob_ugwp_tauamp=0.8e-3      # setting for UGWPv1 non-stationary GWD
@@ -159,9 +159,9 @@ case "${fv3_res}" in
         export tau=4.0
         export rf_cutoff=100.0
         export fv_sg_adj=900
-        export WRITE_GROUP=2
-        export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE=8
-        export WRITE_GROUP_GFS=2
+        export WRITE_GROUP=1
+        export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE=10
+        export WRITE_GROUP_GFS=1
         export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE_GFS=10
         ;;
     "C768")
@@ -357,7 +357,7 @@ if [[ "${skip_mom6}" == "false" ]]; then
       TOPOEDITS=""
       ;;
     "025")
-      ntasks_mom6=220
+      ntasks_mom6=130
       OCNTIM=1800
       NX_GLB=1440
       NY_GLB=1080
@@ -425,7 +425,7 @@ if [[ "${skip_cice6}" == "false" ]]; then
       cice6_processor_shape="slenderX2"
       ;;
     "025")
-      ntasks_cice6=120
+      ntasks_cice6=72
       cice6_processor_shape="slenderX2"
       ;;
     *)
@@ -454,7 +454,8 @@ if [[ "${skip_ww3}" == "false" ]]; then
       ntasks_ww3=100
       ;;
     "glo_025")
-      ntasks_ww3=262
+      ntasks_ww3=250
+      nthreads_ww3=1
       ;;
     "glo_200")
       ntasks_ww3=30

parm/config/gfs/config.base

@@ -270,7 +270,8 @@ fi
 export FHMIN=0
 export FHMAX=9
 export FHOUT=3           # Will be changed to 1 in config.base if (DOHYBVAR set to NO and l4densvar set to false)
-export FHOUT_OCNICE=3
+export FHOUT_OCN=3
+export FHOUT_ICE=3
 
 # Cycle to run EnKF  (set to BOTH for both gfs and gdas)
 export EUPD_CYC="@EUPD_CYC@"
@@ -284,7 +285,8 @@ export FHMAX_GFS=@FHMAX_GFS@
 export FHOUT_GFS=3 # Must be 6 for S2S until #1629 is addressed; 3 for ops
 export FHMAX_HF_GFS=0
 export FHOUT_HF_GFS=1
-export FHOUT_OCNICE_GFS=6
+export FHOUT_OCN_GFS=6
+export FHOUT_ICE_GFS=6   
 if (( gfs_cyc != 0 )); then
     export STEP_GFS=$(( 24 / gfs_cyc ))
 else

parm/config/gfs/config.fcst

@@ -33,7 +33,8 @@ case ${RUN} in
     export FHOUT=${FHOUT_GFS}
     export FHMAX_HF=${FHMAX_HF_GFS}
     export FHOUT_HF=${FHOUT_HF_GFS}
-    export FHOUT_OCNICE=${FHOUT_OCNICE_GFS}
+    export FHOUT_OCN=${FHOUT_OCN_GFS}
+    export FHOUT_ICE=${FHOUT_ICE_GFS}
     ;;
   *gdas)
     export FHMAX_HF=0

parm/post/oceanice_products_gefs.yaml

@@ -0,0 +1,73 @@
+ocnicepost:
+  executable: "ocnicepost.x"
+  namelist:
+    debug: False
+  fix_data:
+    mkdir:
+      - "{{ DATA }}"
+    copy:
+      - ["{{ EXECgfs }}/ocnicepost.x", "{{ DATA }}/"]
+      - ["{{ PARMgfs }}/post/ocnicepost.nml.jinja2", "{{ DATA }}/"]
+      - ["{{ PARMgfs }}/post/{{ component }}.csv", "{{ DATA }}/"]
+      - ["{{ FIXgfs }}/mom6/post/{{ model_grid }}/tripole.{{ model_grid }}.Bu.to.Ct.bilinear.nc", "{{ DATA }}/"]
+      - ["{{ FIXgfs }}/mom6/post/{{ model_grid }}/tripole.{{ model_grid }}.Cu.to.Ct.bilinear.nc", "{{ DATA }}/"]
+      - ["{{ FIXgfs }}/mom6/post/{{ model_grid }}/tripole.{{ model_grid }}.Cv.to.Ct.bilinear.nc", "{{ DATA }}/"]
+      {% for grid in product_grids %}
+      - ["{{ FIXgfs }}/mom6/post/{{ model_grid }}/tripole.{{ model_grid }}.Ct.to.rect.{{ grid }}.bilinear.nc", "{{ DATA }}/"]
+      - ["{{ FIXgfs }}/mom6/post/{{ model_grid }}/tripole.{{ model_grid }}.Ct.to.rect.{{ grid }}.conserve.nc", "{{ DATA }}/"]
+      - ["{{ FIXgfs }}/mom6/post/template.global.{{ grid }}.gb2", "{{ DATA }}/"]
+      {% endfor %}
+
+nc2grib2:
+  script: "{{ USHgfs }}/oceanice_nc2grib2.sh"
+
+ocean:
+  namelist:
+    ftype: "ocean"
+    maskvar: "temp"
+    sinvar: "sin_rot"
+    cosvar: "cos_rot"
+    angvar: ""
+    {% if model_grid == 'mx025' or model_grid == 'mx050' or model_grid == 'mx100' %}
+    ocean_levels: [5, 15, 25, 35, 45, 55, 65, 75, 85, 95, 105, 115, 125, 135, 145, 155, 165, 175, 185, 195, 205, 215, 226, 241, 267, 309, 374, 467, 594, 757, 960, 1204, 1490, 1817, 2184, 2587, 3024, 3489, 3977, 4481]
+    {% elif model_grid == 'mx500' %}
+    ocean_levels: [5, 15, 25, 35, 45, 55, 65, 75, 85, 95, 105, 115, 125, 135, 145, 155, 165, 175, 185, 195, 205, 215, 226, 241, 267]
+    {% endif %}
+  subset: ['SSH', 'SST', 'SSS', 'speed', 'MLD_003', 'latent', 'sensible', 'SW', 'LW', 'LwLatSens', 'Heat_PmE', 'SSU', 'SSV', 'taux', 'tauy', 'temp', 'so', 'uo', 'vo']
+  data_in:
+    copy:
+      - ["{{ COM_OCEAN_HISTORY }}/{{ RUN }}.ocean.t{{ current_cycle | strftime('%H') }}z.{{ interval }}hr_avg.f{{ '%03d' % forecast_hour }}.nc", "{{ DATA }}/ocean.nc"]
+  data_out:
+    mkdir:
+      - "{{ COM_OCEAN_NETCDF }}"
+      {% for grid in product_grids %}
+      - "{{ COM_OCEAN_GRIB }}/{{ grid }}"
+      {% endfor %}
+    copy:
+      - ["{{ DATA }}/ocean_subset.nc", "{{ COM_OCEAN_NETCDF }}/{{ RUN }}.ocean.t{{ current_cycle | strftime('%H') }}z.native.f{{ '%03d' % forecast_hour }}.nc"]
+      {% for grid in product_grids %}
+      - ["{{ DATA }}/ocean.{{ grid }}.nc", "{{ COM_OCEAN_NETCDF }}/{{ RUN }}.ocean.t{{ current_cycle | strftime('%H') }}z.{{ grid }}.f{{ '%03d' % forecast_hour }}.nc"]
+      {% endfor %}
+
+ice:
+  namelist:
+    ftype: "ice"
+    maskvar: "tmask"
+    sinvar: ""
+    cosvar: ""
+    angvar: "ANGLET"
+  subset: ['hi_h', 'hs_h', 'aice_h', 'Tsfc_h', 'uvel_h', 'vvel_h', 'frzmlt_h', 'albsni_h', 'mlt_onset_h', 'frz_onset_h']
+  data_in:
+    copy:
+      - ["{{ COM_ICE_HISTORY }}/{{ RUN }}.ice.t{{ current_cycle | strftime('%H') }}z.{{ interval }}hr_avg.f{{ '%03d' % forecast_hour }}.nc", "{{ DATA }}/ice.nc"]
+  data_out:
+    mkdir:
+      - "{{ COM_ICE_NETCDF }}"
+      {% for grid in product_grids %}
+      - "{{ COM_ICE_GRIB }}/{{ grid }}"
+      {% endfor %}
+    copy:
+      - ["{{ DATA }}/ice_subset.nc", "{{ COM_ICE_NETCDF }}/{{ RUN }}.ice.t{{ current_cycle | strftime('%H') }}z.native.f{{ '%03d' % forecast_hour }}.nc"]
+      {% for grid in product_grids %}
+      - ["{{ DATA }}/ice.{{ grid }}.nc", "{{ COM_ICE_NETCDF }}/{{ RUN }}.ice.t{{ current_cycle | strftime('%H') }}z.{{ grid }}.f{{ '%03d' % forecast_hour }}.nc"]
+      {% endfor %}

parm/ufs/fv3/diag_table

@@ -1,6 +1,6 @@
 "fv3_history",    0,  "hours",  1,  "hours",  "time"
 "fv3_history2d",  0,  "hours",  1,  "hours",  "time"
-"@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", @[FHOUT_OCNICE],  "hours",  1,  "hours",  "time",  @[FHOUT_OCNICE],  "hours",  "@[SYEAR] @[SMONTH] @[SDAY] @[CHOUR] 0 0"
+"@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", @[FHOUT_OCN],  "hours",  1,  "hours",  "time",  @[FHOUT_OCN],  "hours",  "@[SYEAR] @[SMONTH] @[SDAY] @[CHOUR] 0 0"
 "@[MOM6_OUTPUT_DIR]/ocn_daily%4yr%2mo%2dy",  1,  "days",   1,  "days",   "time",  1,  "days",   "@[SYEAR] @[SMONTH] @[SDAY] @[CHOUR] 0 0"
 
 ##############

ush/check_ice_netcdf.sh

@@ -0,0 +1,38 @@
+#! /usr/bin/env bash
+
+# shellcheck disable=SC2155,SC2312
+HOMEgfs=$(cd "$(dirname "$(readlink -f -n "${BASH_SOURCE[0]}" )" )/.." && pwd -P)
+declare -rx HOMEgfs
+
+source "${HOMEgfs}/ush/load_fv3gfs_modules.sh" 1>/dev/null 2>&1
+source "${HOMEgfs}/ush/jjob_header.sh" -e "oceanice_products" -c "base oceanice_products" 1>/dev/null 2>&1
+
+yyyy=${1?}
+mm=${2?}
+dd=${3?}
+cyc=${4?}
+fhr=${5?}
+ROTDIR=${6?}
+member=${7?}
+
+fhri=$(echo $fhr | sed 's/^0*//')
+
+if [[ "${cyc}" != "00" ]] && (( FHOUT_ICE_GFS % 24 == 0 ));then
+  (( fhri = ${fhri} - cyc ))
+  fhr3=$(printf %03i "${fhri}")
+  if (( fhri <= 24  )); then
+    (( interval = 24 - cyc )) 
+    ncfile=$ROTDIR/gefs.${yyyy}${mm}${dd}/${cyc}/mem${member}/model_data/ice/history/gefs.ice.t${cyc}z.${interval}hr_avg.f${fhr3}.nc
+  else
+    ncfile=$ROTDIR/gefs.${yyyy}${mm}${dd}/${cyc}/mem${member}/model_data/ice/history/gefs.ice.t${cyc}z.${FHOUT_ICE_GFS}hr_avg.f${fhr3}.nc
+  fi
+else
+  ncfile=$ROTDIR/gefs.${yyyy}${mm}${dd}/${cyc}/mem${member}/model_data/ice/history/gefs.ice.t${cyc}z.${FHOUT_ICE_GFS}hr_avg.f${fhr}.nc
+fi
+
+ls -l $ncfile 2>&1  # redirect stdout and stderr to /dev/null to suppress output in cron
+
+rc=$?
+# If there is no error, rc=0, else rc!=0
+
+exit "${rc}"

ush/forecast_postdet.sh

@@ -543,29 +543,57 @@ CICE_postdet() {
   # Link CICE forecast output files from DATA/CICE_OUTPUT to COM
   local source_file dest_file
   for fhr in ${CICE_OUTPUT_FH}; do
-    fhr3=$(printf %03i "${fhr}")
 
     if [[ -z ${last_fhr:-} ]]; then
       last_fhr=${fhr}
       continue
     fi
 
-    (( interval = fhr - last_fhr ))
+    if [[ ${cyc} != "00" ]]  && (( FHOUT_ICE % 24 == 0 )); then
+      if [[ ${fhr} -le 24 ]];then
+        (( fhr = 24 - cyc ))
+      else
+        (( fhr = last_fhr + 24 - cyc ))
+      fi
+    fi
+
+    fhr3=$(printf %03i "${fhr}")
+
+    if [[ "${cyc}" != "00" ]] && (( FHOUT_ICE % 24 == 0 )); then
+      if [[  ${fhr} -le 24 ]]; then
+        interval=${cyc}
+        interval_source=24
+      else
+        interval=24
+        interval_source=${interval}
+      fi
+    else
+        (( interval = fhr - last_fhr ))
+    fi
 
     vdate=$(date --utc -d "${current_cycle:0:8} ${current_cycle:8:2} + ${fhr} hours" +%Y%m%d%H)
     seconds=$(to_seconds "${vdate:8:2}0000")  # convert HHMMSS to seconds
     vdatestr="${vdate:0:4}-${vdate:4:2}-${vdate:6:2}-${seconds}"
 
     if [[ "${RUN}" =~ "gfs" || "${RUN}" =~ "gefs" ]]; then
-      source_file="iceh_$(printf "%0.2d" "${interval}")h.${vdatestr}.nc"
+      if [[ "${cyc}" != "00" ]] && (( FHOUT_ICE % 24 == 0 )); then
+        source_file="iceh_$(printf "%0.2d" "${interval_source}")h.${vdatestr}.nc"
+      else
+        source_file="iceh_$(printf "%0.2d" "${interval}")h.${vdatestr}.nc"
+      fi
       dest_file="${RUN}.ice.t${cyc}z.${interval}hr_avg.f${fhr3}.nc"
     elif [[ "${RUN}" =~ "gdas" ]]; then
       source_file="iceh_inst.${vdatestr}.nc"
       dest_file="${RUN}.ice.t${cyc}z.inst.f${fhr3}.nc"
     fi
     ${NLN} "${COM_ICE_HISTORY}/${dest_file}" "${DATA}/CICE_OUTPUT/${source_file}"
 
-    last_fhr=${fhr}
+
+     if [[ ${cyc} != "00" ]] && (( FHOUT_ICE % 24 == 0 )); then
+       (( last_fhr = fhr + cyc ))
+     else
+       last_fhr=${fhr}
+     fi
   done
 
 }

ush/forecast_predet.sh

@@ -568,7 +568,7 @@ CICE_predet(){
 
   # CICE does not have a concept of high frequency output like FV3
   # Convert output settings into an explicit list for CICE
-  CICE_OUTPUT_FH=$(seq -s ' ' "${FHMIN}" "${FHOUT_OCNICE}" "${FHMAX}")
+  CICE_OUTPUT_FH=$(seq -s ' ' "${FHMIN}" "${FHOUT_ICE}" "${FHMAX}")
 
   # Fix files
   ${NCP} "${FIXgfs}/cice/${ICERES}/${CICE_GRID}" "${DATA}/"
@@ -591,7 +591,7 @@ MOM6_predet(){
 
   # MOM6 does not have a concept of high frequency output like FV3
   # Convert output settings into an explicit list for MOM6
-  MOM6_OUTPUT_FH=$(seq -s ' ' "${FHMIN}" "${FHOUT_OCNICE}" "${FHMAX}")
+  MOM6_OUTPUT_FH=$(seq -s ' ' "${FHMIN}" "${FHOUT_OCN}" "${FHMAX}")
 
   # If using stochastic parameterizations, create a seed that does not exceed the
   # largest signed integer

ush/parsing_namelists_CICE.sh

@@ -62,7 +62,7 @@ local CICE_RESTART_FILE="cice_model.res"
 local CICE_DUMPFREQ="y"  # "h","d","m" or "y" for restarts at intervals of "hours", "days", "months" or "years"
 local CICE_DUMPFREQ_N=10000  # Set this to a really large value, as cice, mom6 and cmeps restart interval is controlled by ufs.configure
 local CICE_DIAGFREQ=$(( 86400 / DT_CICE ))  # frequency of diagnostic output in timesteps, recommended for 1x per day
-local CICE_HISTFREQ_N="0, 0, ${FHOUT_OCNICE}, 1, 1"
+local CICE_HISTFREQ_N="0, 0, ${FHOUT_ICE}, 1, 1"
 if [[ "${RUN}" =~ "gdas" ]]; then
   local CICE_HIST_AVG=".false., .false., .false., .false., .false."   # DA needs instantaneous
 else