diff --git a/ci/cases/pr/C48mx500_3DVarAOWCDA.yaml b/ci/cases/pr/C48mx500_3DVarAOWCDA.yaml new file mode 100644 index 0000000000..b972d3a445 --- /dev/null +++ b/ci/cases/pr/C48mx500_3DVarAOWCDA.yaml @@ -0,0 +1,22 @@ +experiment: + system: gfs + mode: cycled + +arguments: + pslot: {{ 'pslot' | getenv }} + app: S2S + resdetatmos: 48 + resdetocean: 5.0 + comroot: {{ 'RUNTESTS' | getenv }}/COMROOT + expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR + icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C48mx500 + idate: 2021032412 + edate: 2021032418 + nens: 0 + gfs_cyc: 0 + start: warm + yaml: {{ HOMEgfs }}/ci/cases/yamls/soca_gfs_defaults_ci.yaml + +skip_ci_on_hosts: + - orion + - hercules diff --git a/ci/cases/yamls/soca_gfs_defaults_ci.yaml b/ci/cases/yamls/soca_gfs_defaults_ci.yaml new file mode 100644 index 0000000000..126637cd86 --- /dev/null +++ b/ci/cases/yamls/soca_gfs_defaults_ci.yaml @@ -0,0 +1,5 @@ +defaults: + !INC {{ HOMEgfs }}/parm/config/gfs/yaml/defaults.yaml +base: + ACCOUNT: {{ 'SLURM_ACCOUNT' | getenv }} + DO_JEDIOCNVAR: "YES" diff --git a/env/HERA.env b/env/HERA.env index e02c0aad22..057a2313f8 100755 --- a/env/HERA.env +++ b/env/HERA.env @@ -93,31 +93,19 @@ elif [[ "${step}" = "ocnanalbmat" ]]; then export APRUNCFP="${launcher} -n \$ncmd --multi-prog" - nth_max=$((npe_node_max / npe_node_ocnanalbmat)) - - export NTHREADS_OCNANAL=${nth_ocnanalbmat:-${nth_max}} - [[ ${NTHREADS_OCNANAL} -gt ${nth_max} ]] && export NTHREADS_OCNANAL=${nth_max} - export APRUN_OCNANAL="${launcher} -n ${npe_ocnanalbmat} --cpus-per-task=${NTHREADS_OCNANAL}" + export APRUN_OCNANAL="${launcher} -n ${npe_ocnanalbmat}" elif [[ "${step}" = "ocnanalrun" ]]; then export APRUNCFP="${launcher} -n \$ncmd --multi-prog" - nth_max=$((npe_node_max / npe_node_ocnanalrun)) - - export NTHREADS_OCNANAL=${nth_ocnanalrun:-${nth_max}} - [[ ${NTHREADS_OCNANAL} -gt ${nth_max} ]] && export NTHREADS_OCNANAL=${nth_max} - export APRUN_OCNANAL="${launcher} -n ${npe_ocnanalrun} --cpus-per-task=${NTHREADS_OCNANAL}" + export APRUN_OCNANAL="${launcher} -n ${npe_ocnanalrun}" elif [[ "${step}" = "ocnanalchkpt" ]]; then export APRUNCFP="${launcher} -n \$ncmd --multi-prog" - nth_max=$((npe_node_max / npe_node_ocnanalchkpt)) - - export NTHREADS_OCNANAL=${nth_ocnanalchkpt:-${nth_max}} - [[ ${NTHREADS_OCNANAL} -gt ${nth_max} ]] && export NTHREADS_OCNANAL=${nth_max} - export APRUN_OCNANAL="${launcher} -n ${npe_ocnanalchkpt} --cpus-per-task=${NTHREADS_OCNANAL}" + export APRUN_OCNANAL="${launcher} -n ${npe_ocnanalchkpt}" elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then diff --git a/jobs/JGLOBAL_PREP_OCEAN_OBS b/jobs/JGLOBAL_PREP_OCEAN_OBS index a100aca89c..a6fcf9c9b3 100755 --- a/jobs/JGLOBAL_PREP_OCEAN_OBS +++ b/jobs/JGLOBAL_PREP_OCEAN_OBS @@ -15,7 +15,7 @@ YMD=${PDY} HH=${cyc} generate_com -rx COMOUT_OBS:COM_OBS_TMPL ############################################## # Add prep_marine_obs.py to PYTHONPATH -export PYTHONPATH=${HOMEgfs}/sorc/gdas.cd/ush/soca:${PYTHONPATH} +export PYTHONPATH=${HOMEgfs}/sorc/gdas.cd/ush:${PYTHONPATH} ############################################################### # Run relevant script @@ -38,7 +38,7 @@ if [[ -e "${pgmout}" ]] ; then fi ########################################## -# Handle the temporary working directory +# Handle the temporary working directory ########################################## cd "${DATAROOT}" || (echo "FATAL ERROR: ${DATAROOT} does not exist. ABORT!"; exit 1) [[ ${KEEPDATA} = "NO" ]] && rm -rf "${DATA}" diff --git a/parm/config/gfs/config.base.emc.dyn b/parm/config/gfs/config.base.emc.dyn index 5260bbd4f6..6e1e8339f0 100644 --- a/parm/config/gfs/config.base.emc.dyn +++ b/parm/config/gfs/config.base.emc.dyn @@ -378,6 +378,7 @@ export binary_diag=".false." # Verification options export DO_METP="NO" # Run METPLUS jobs - set METPLUS settings in config.metp; not supported with spack-stack export DO_FIT2OBS="YES" # Run fit to observations package +export DO_VRFY_OCEANDA="NO" # Run SOCA Ocean DA verification tasks # Archiving options export HPSSARCH="@HPSSARCH@" # save data to HPSS archive diff --git a/parm/config/gfs/config.ocn b/parm/config/gfs/config.ocn index f9e6595ce9..317a76e58a 100644 --- a/parm/config/gfs/config.ocn +++ b/parm/config/gfs/config.ocn @@ -16,6 +16,14 @@ if [[ "${DO_JEDIOCNVAR}" == "YES" ]]; then else export ODA_INCUPD="False" fi -export ODA_INCUPD_NHOURS="3.0" # In MOM_input, this is time interval for applying increment + +# Time interval for applying the increment +if [[ "${DOIAU}" == "YES" ]]; then + export ODA_INCUPD_NHOURS="6.0" +else + export ODA_INCUPD_NHOURS="3.0" +fi + + echo "END: config.ocn" diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index 1dcf757de2..b746a4b32a 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -352,7 +352,7 @@ case ${step} in export npe_prepoceanobs=1 export nth_prepoceanobs=1 export npe_node_prepoceanobs=$(( npe_node_max / nth_prepoceanobs )) - export memory_prepoceanobs="24GB" + export memory_prepoceanobs="48GB" ;; "ocnanalbmat") @@ -406,13 +406,20 @@ case ${step} in export nth_ocnanalchkpt=1 export npe_node_ocnanalchkpt=$(( npe_node_max / nth_ocnanalchkpt )) case ${CASE} in - "C384") memory_ocnanalchkpt="128GB";; - "C96") memory_ocnanalchkpt="32GB";; - "C48") memory_ocnanalchkpt="32GB";; + "C384") + memory_ocnanalchkpt="128GB" + npes=40;; + "C96") + memory_ocnanalchkpt="32GB" + npes=16;; + "C48") + memory_ocnanalchkpt="32GB" + npes=8;; *) echo "FATAL ERROR: Resources not defined for job ${job} at resolution ${CASE}" exit 4 esac + export npe_ocnanalchkpt=${npes} export memory_ocnanalchkpt ;; @@ -926,7 +933,7 @@ case ${step} in export nth_eupd=4 fi ;; - *) + *) echo "FATAL ERROR: Resources not defined for job ${job} at resolution ${CASE}" exit 4 ;; diff --git a/parm/config/gfs/yaml/defaults.yaml b/parm/config/gfs/yaml/defaults.yaml index 61fc32c126..10af47de07 100644 --- a/parm/config/gfs/yaml/defaults.yaml +++ b/parm/config/gfs/yaml/defaults.yaml @@ -25,13 +25,14 @@ landanl: IO_LAYOUT_Y: 1 ocnanal: - SOCA_INPUT_FIX_DIR: "/scratch2/NCEPDEV/ocean/Guillaume.Vernieres/data/static/72x35x25/soca" # TODO: These need to go to glopara fix space. @guillaumevernieres will open an issue - CASE_ANL: "C48" - COMIN_OBS: "/scratch2/NCEPDEV/marineda/r2d2-v2-v3" # TODO: make platform agnostic - SOCA_OBS_LIST: "{{ HOMEgfs }}/sorc/gdas.cd/parm/soca/obs/obs_list.yaml" + SOCA_INPUT_FIX_DIR: "/scratch2/NCEPDEV/ocean/Guillaume.Vernieres/data/static/72x35x25/soca" # TODO: These need to go to glopara fix space. + CASE_ANL: "C48" # TODO: Check in gdasapp if used anywhere for SOCA + SOCA_OBS_LIST: "{{ HOMEgfs }}/sorc/gdas.cd/parm/soca/obs/obs_list.yaml" # TODO: This is also repeated in oceanprepobs SOCA_NINNER: 100 - R2D2_OBS_SRC: "gdas_marine" - R2D2_OBS_DUMP: "s2s_v1" SABER_BLOCKS_YAML: "" NICAS_RESOL: 1 NICAS_GRID_SIZE: 15000 +prepoceanobs: + SOCA_OBS_LIST: "{{ HOMEgfs }}/sorc/gdas.cd/parm/soca/obs/obs_list.yaml" # TODO: This is also repeated in ocnanal + OBSPREP_YAML: "{{ HOMEgfs }}/sorc/gdas.cd/parm/soca/obsprep/obsprep_config.yaml" + DMPDIR: "/scratch1/NCEPDEV/global/glopara/data/experimental_obs" diff --git a/workflow/applications/gfs_cycled.py b/workflow/applications/gfs_cycled.py index 7b5892d7c0..6dd0342a78 100644 --- a/workflow/applications/gfs_cycled.py +++ b/workflow/applications/gfs_cycled.py @@ -18,6 +18,7 @@ def __init__(self, conf: Configuration): self.do_jediocnvar = self._base.get('DO_JEDIOCNVAR', False) self.do_jedilandda = self._base.get('DO_JEDILANDDA', False) self.do_mergensst = self._base.get('DO_MERGENSST', False) + self.do_vrfy_oceanda = self._base.get('DO_VRFY_OCEANDA', False) self.lobsdiag_forenkf = False self.eupd_cdumps = None @@ -43,8 +44,9 @@ def _get_app_configs(self): if self.do_jediocnvar: configs += ['prepoceanobs', 'ocnanalprep', 'ocnanalbmat', - 'ocnanalrun', 'ocnanalchkpt', 'ocnanalpost', - 'ocnanalvrfy'] + 'ocnanalrun', 'ocnanalchkpt', 'ocnanalpost'] + if self.do_vrfy_oceanda: + configs += ['ocnanalvrfy'] if self.do_ocean: configs += ['ocnpost'] @@ -137,8 +139,9 @@ def get_task_names(self): if self.do_jediocnvar: gdas_gfs_common_tasks_before_fcst += ['prepoceanobs', 'ocnanalprep', 'ocnanalbmat', 'ocnanalrun', - 'ocnanalchkpt', 'ocnanalpost', - 'ocnanalvrfy'] + 'ocnanalchkpt', 'ocnanalpost'] + if self.do_vrfy_oceanda: + gdas_gfs_common_tasks_before_fcst += ['ocnanalvrfy'] gdas_gfs_common_tasks_before_fcst += ['sfcanl', 'analcalc'] diff --git a/workflow/setup_expt.py b/workflow/setup_expt.py index 2bc41854d8..3eeb584f46 100755 --- a/workflow/setup_expt.py +++ b/workflow/setup_expt.py @@ -224,6 +224,13 @@ def link_files_from_src_to_dst(src_dir, dst_dir): src_file = os.path.join(src_dir, fname) if os.path.exists(src_file): os.symlink(src_file, os.path.join(dst_dir, fname)) + # First 1/2 cycle also needs a atmos increment if doing warm start + if inputs.start in ['warm']: + for ftype in ['atmi003.nc', 'atminc.nc', 'atmi009.nc']: + fname = f'{inputs.cdump}.t{idatestr[8:]}z.{ftype}' + src_file = os.path.join(src_dir, fname) + if os.path.exists(src_file): + os.symlink(src_file, os.path.join(dst_dir, fname)) return