From 754fc290a002764ca50dbedc6d67d9baa73cb6cc Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA Date: Tue, 19 Dec 2023 20:57:26 +0000 Subject: [PATCH 1/7] add COMOUT_OBS to marine obs prep --- jobs/JGLOBAL_PREP_OCEAN_OBS | 1 + 1 file changed, 1 insertion(+) diff --git a/jobs/JGLOBAL_PREP_OCEAN_OBS b/jobs/JGLOBAL_PREP_OCEAN_OBS index 44cbbf1c08..a100aca89c 100755 --- a/jobs/JGLOBAL_PREP_OCEAN_OBS +++ b/jobs/JGLOBAL_PREP_OCEAN_OBS @@ -8,6 +8,7 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "prepoceanobs" -c "base prepoceanobs" ############################################## export COMIN_OBS="${DATA}" +YMD=${PDY} HH=${cyc} generate_com -rx COMOUT_OBS:COM_OBS_TMPL ############################################## # Begin JOB SPECIFIC work From 6e49d519a1e34bf3a6244d70a5dcea99f612eae4 Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA Date: Mon, 8 Jan 2024 15:56:54 +0000 Subject: [PATCH 2/7] adding prepoceanobs to workflow --- parm/config/gfs/config.ocnanal | 10 +++++----- workflow/applications/gfs_cycled.py | 10 +++++++--- workflow/rocoto/gfs_tasks.py | 18 +++++++++++++++++- workflow/rocoto/tasks.py | 1 + 4 files changed, 30 insertions(+), 9 deletions(-) diff --git a/parm/config/gfs/config.ocnanal b/parm/config/gfs/config.ocnanal index ec45ddd288..7506cf23fd 100644 --- a/parm/config/gfs/config.ocnanal +++ b/parm/config/gfs/config.ocnanal @@ -17,11 +17,11 @@ export CASE_ANL=@CASE_ANL@ export DOMAIN_STACK_SIZE=116640000 #TODO: Make the stack size resolution dependent export JEDI_BIN=${HOMEgfs}/sorc/gdas.cd/build/bin -# R2D2 -export R2D2_OBS_DB=shared -export R2D2_OBS_DUMP=@R2D2_OBS_DUMP@ -export R2D2_OBS_SRC=@R2D2_OBS_SRC@ -export R2D2_OBS_WINDOW=24 # TODO: Check if the R2D2 sampling DB window is still needed +## R2D2 +#export R2D2_OBS_DB=shared +#export R2D2_OBS_DUMP=@R2D2_OBS_DUMP@ +#export R2D2_OBS_SRC=@R2D2_OBS_SRC@ +#export R2D2_OBS_WINDOW=24 # TODO: Check if the R2D2 sampling DB window is still needed export COMIN_OBS=@COMIN_OBS@ # NICAS diff --git a/workflow/applications/gfs_cycled.py b/workflow/applications/gfs_cycled.py index 29c6b18f43..49fd4aac67 100644 --- a/workflow/applications/gfs_cycled.py +++ b/workflow/applications/gfs_cycled.py @@ -42,7 +42,9 @@ def _get_app_configs(self): configs += ['anal', 'analdiag'] if self.do_jediocnvar: - configs += ['ocnanalprep', 'ocnanalbmat', 'ocnanalrun', 'ocnanalchkpt', 'ocnanalpost', 'ocnanalvrfy'] + configs += [ 'prepoceanobs', 'ocnanalprep', 'ocnanalbmat', + 'ocnanalrun', 'ocnanalchkpt', 'ocnanalpost', + 'ocnanalvrfy'] if self.do_ocean: configs += ['ocnpost'] @@ -133,8 +135,10 @@ def get_task_names(self): gdas_gfs_common_tasks_before_fcst += ['anal'] if self.do_jediocnvar: - gdas_gfs_common_tasks_before_fcst += ['ocnanalprep', 'ocnanalbmat', 'ocnanalrun', - 'ocnanalchkpt', 'ocnanalpost', 'ocnanalvrfy'] + gdas_gfs_common_tasks_before_fcst += ['prepoceanobs', 'ocnanalprep', + 'ocnanalbmat', 'ocnanalrun', + 'ocnanalchkpt', 'ocnanalpost', + 'ocnanalvrfy'] gdas_gfs_common_tasks_before_fcst += ['sfcanl', 'analcalc'] diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 18208983b8..e87ac1bfbd 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -364,7 +364,7 @@ def landanl(self): task = create_wf_task('landanl', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) return task - def ocnanalprep(self): + def prepoceanobs(self): ocean_hist_path = self._template_to_rocoto_cycstring(self._base["COM_OCEAN_HISTORY_TMPL"], {'RUN': 'gdas'}) @@ -374,6 +374,22 @@ def ocnanalprep(self): deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) + resources = self.get_resource('prepoceanobs') + task = create_wf_task('prepoceanobs', + resources, + cdump=self.cdump, + envar=self.envars, + dependency=dependencies) + + return task + + def ocnanalprep(self): + + deps = [] + dep_dict = {'type': 'task', 'name': f'{self.cdump}prepoceanobs'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) + resources = self.get_resource('ocnanalprep') task = create_wf_task('ocnanalprep', resources, diff --git a/workflow/rocoto/tasks.py b/workflow/rocoto/tasks.py index 2d44c00d4d..7deb2bef35 100644 --- a/workflow/rocoto/tasks.py +++ b/workflow/rocoto/tasks.py @@ -13,6 +13,7 @@ class Tasks: VALID_TASKS = ['aerosol_init', 'stage_ic', 'prep', 'anal', 'sfcanl', 'analcalc', 'analdiag', 'arch', "cleanup", 'prepatmiodaobs', 'atmanlinit', 'atmanlrun', 'atmanlfinal', + 'prepoceanobs', 'ocnanalprep', 'ocnanalbmat', 'ocnanalrun', 'ocnanalchkpt', 'ocnanalpost', 'ocnanalvrfy', 'earc', 'ecen', 'echgres', 'ediag', 'efcs', 'eobs', 'eomg', 'epos', 'esfc', 'eupd', From 4a5cd4e6e382ad24d19ee80572921d407f001f1f Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA Date: Wed, 10 Jan 2024 20:34:01 +0000 Subject: [PATCH 3/7] changes to use ocean obs prep in cycling and remove r2d2 --- parm/config/gfs/config.ocnanal | 5 ----- parm/config/gfs/config.prepoceanobs | 5 ++++- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/parm/config/gfs/config.ocnanal b/parm/config/gfs/config.ocnanal index 7506cf23fd..38a6cbd52a 100644 --- a/parm/config/gfs/config.ocnanal +++ b/parm/config/gfs/config.ocnanal @@ -17,11 +17,6 @@ export CASE_ANL=@CASE_ANL@ export DOMAIN_STACK_SIZE=116640000 #TODO: Make the stack size resolution dependent export JEDI_BIN=${HOMEgfs}/sorc/gdas.cd/build/bin -## R2D2 -#export R2D2_OBS_DB=shared -#export R2D2_OBS_DUMP=@R2D2_OBS_DUMP@ -#export R2D2_OBS_SRC=@R2D2_OBS_SRC@ -#export R2D2_OBS_WINDOW=24 # TODO: Check if the R2D2 sampling DB window is still needed export COMIN_OBS=@COMIN_OBS@ # NICAS diff --git a/parm/config/gfs/config.prepoceanobs b/parm/config/gfs/config.prepoceanobs index 068ecff1ad..bde58ec9c1 100644 --- a/parm/config/gfs/config.prepoceanobs +++ b/parm/config/gfs/config.prepoceanobs @@ -7,11 +7,14 @@ echo "BEGIN: config.prepoceanobs" export OCNOBS2IODAEXEC=${HOMEgfs}/sorc/gdas.cd/build/bin/gdas_obsprovider2ioda.x export OBS_YAML_DIR=${HOMEgfs}/sorc/gdas.cd/parm/soca/obs/config -export OBSPROC_YAML=@OBSPROC_CONFIG@ +export OBSPROC_YAML=${HOMEgfs}/sorc/gdas.cd/parm/soca/obsproc/obsproc_config.yaml export OBS_LIST=@SOCA_OBS_LIST@ [[ -n "${OBS_LIST}" ]] || export OBS_LIST=${HOMEgfs}/sorc/gdas.cd/parm/soca/obs/obs_list.yaml export OBS_YAML=${OBS_LIST} +# until dmpdir has full ocean obs; works on Hera +export DMPDIR=/scratch1/NCEPDEV/stmp4/Shastri.Paturi/forAndrew + # Get task specific resources . "${EXPDIR}/config.resources" prepoceanobs echo "END: config.prepoceanobs" From efab33fea7f01c083eea2232dd9c3e964285399b Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA Date: Wed, 10 Jan 2024 20:44:55 +0000 Subject: [PATCH 4/7] coding norms correction --- workflow/applications/gfs_cycled.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow/applications/gfs_cycled.py b/workflow/applications/gfs_cycled.py index 5b8400326d..e96c3171c6 100644 --- a/workflow/applications/gfs_cycled.py +++ b/workflow/applications/gfs_cycled.py @@ -42,7 +42,7 @@ def _get_app_configs(self): configs += ['anal', 'analdiag'] if self.do_jediocnvar: - configs += [ 'prepoceanobs', 'ocnanalprep', 'ocnanalbmat', + configs += ['prepoceanobs', 'ocnanalprep', 'ocnanalbmat', 'ocnanalrun', 'ocnanalchkpt', 'ocnanalpost', 'ocnanalvrfy'] From 949e3cb8ca9bdc9e134df68c977a799543238bd7 Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA Date: Wed, 10 Jan 2024 20:46:07 +0000 Subject: [PATCH 5/7] coding norms correction --- workflow/applications/gfs_cycled.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/workflow/applications/gfs_cycled.py b/workflow/applications/gfs_cycled.py index e96c3171c6..1ff6cc3723 100644 --- a/workflow/applications/gfs_cycled.py +++ b/workflow/applications/gfs_cycled.py @@ -43,8 +43,8 @@ def _get_app_configs(self): if self.do_jediocnvar: configs += ['prepoceanobs', 'ocnanalprep', 'ocnanalbmat', - 'ocnanalrun', 'ocnanalchkpt', 'ocnanalpost', - 'ocnanalvrfy'] + 'ocnanalrun', 'ocnanalchkpt', 'ocnanalpost', + 'ocnanalvrfy'] if self.do_ocean: configs += ['ocnpost'] From a68b327ce5821fedf511c6619991b5bf8797e1bf Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA Date: Thu, 11 Jan 2024 15:50:47 +0000 Subject: [PATCH 6/7] make OBSPROC_YAML and DMPDIR setup-determined --- parm/config/gfs/config.prepoceanobs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/parm/config/gfs/config.prepoceanobs b/parm/config/gfs/config.prepoceanobs index bde58ec9c1..d7c4e37bb9 100644 --- a/parm/config/gfs/config.prepoceanobs +++ b/parm/config/gfs/config.prepoceanobs @@ -7,13 +7,13 @@ echo "BEGIN: config.prepoceanobs" export OCNOBS2IODAEXEC=${HOMEgfs}/sorc/gdas.cd/build/bin/gdas_obsprovider2ioda.x export OBS_YAML_DIR=${HOMEgfs}/sorc/gdas.cd/parm/soca/obs/config -export OBSPROC_YAML=${HOMEgfs}/sorc/gdas.cd/parm/soca/obsproc/obsproc_config.yaml +export OBSPROC_YAML=@OBSPROC_YAML@ export OBS_LIST=@SOCA_OBS_LIST@ [[ -n "${OBS_LIST}" ]] || export OBS_LIST=${HOMEgfs}/sorc/gdas.cd/parm/soca/obs/obs_list.yaml export OBS_YAML=${OBS_LIST} -# until dmpdir has full ocean obs; works on Hera -export DMPDIR=/scratch1/NCEPDEV/stmp4/Shastri.Paturi/forAndrew +# ocean analysis needs own dmpdir until standard dmpdir has full ocean obs +export DMPDIR=@DMPDIR@ # Get task specific resources . "${EXPDIR}/config.resources" prepoceanobs From 61a18c58fe78361f6e5606d4639dd6e33514a012 Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA Date: Fri, 12 Jan 2024 13:21:16 +0000 Subject: [PATCH 7/7] bugfixes --- workflow/rocoto/gfs_tasks.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 22b76d9936..0f5e184192 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -589,11 +589,19 @@ def prepoceanobs(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('prepoceanobs') - task = create_wf_task('prepoceanobs', - resources, - cdump=self.cdump, - envar=self.envars, - dependency=dependencies) + task_name = f'{self.cdump}prepoceanobs' + task_dict = {'task_name': task_name, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars, + 'cycledef': self.cdump.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/prepoceanobs.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' + } + + task = rocoto.create_task(task_dict) return task @@ -653,13 +661,13 @@ def ocnanalrun(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('ocnanalrun') - task_name = f'{self.cdump}ocnanlrun' + task_name = f'{self.cdump}ocnanalrun' task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.cdump.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/ocnanlrun.sh', + 'command': f'{self.HOMEgfs}/jobs/rocoto/ocnanalrun.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;'