Update UFSDA ATM anl for new COM directory structure (#1537)

g-w PR #1421 changed the GFS COM directory structure. This PR updates UFSDA ATM variational analysis jobs and python scripts to work with the updated GFS COM directory structure. Fixes #1517
NOAA-EMC · Apr 26, 2023 · 3dd6bbe · 3dd6bbe
1 parent b5d173f
commit 3dd6bbe
Show file tree

Hide file tree

Showing 5 changed files with 44 additions and 34 deletions.
diff --git a/jobs/JGLOBAL_ATM_ANALYSIS_FINALIZE b/jobs/JGLOBAL_ATM_ANALYSIS_FINALIZE
@@ -8,20 +8,27 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "atmanlfinal" -c "base atmanl atmanlfi
 ##############################################
 # Set variables used in the script
 ##############################################
+# shellcheck disable=SC2153
 GDATE=$(date +%Y%m%d%H -d "${PDY} ${cyc} - ${assim_freq} hours")
+gPDY=${GDATE:0:8}
+gcyc=${GDATE:8:2}
 GDUMP="gdas"
 
 
 ##############################################
 # Begin JOB SPECIFIC work
 ##############################################
 
-export COMOUT=${COMOUT:-${ROTDIR}/${RUN}.${PDY}/${cyc}/atmos}
-mkdir -p "${COMOUT}"
+# Generate COM variables from templates
+YMD=${PDY} HH=${cyc} generate_com -rx COM_ATMOS_ANALYSIS
+
+RUN=${GDUMP} YMD=${gPDY} HH=${gcyc} generate_com -rx \
+    COM_ATMOS_ANALYSIS_PREV:COM_ATMOS_ANALYSIS_TMPL \
+    COM_ATMOS_HISTORY_PREV:COM_ATMOS_HISTORY_TMPL \
+    COM_ATMOS_RESTART_PREV:COM_ATMOS_RESTART_TMPL
+
+mkdir -m 775 -p "${COM_ATMOS_ANALYSIS}"
 
-# COMIN_GES and COMIN_GES_ENS are used in script
-export COMIN_GES="${ROTDIR}/${GDUMP}.${GDATE:0:8}/${GDATE:8:2}/atmos"
-export COMIN_GES_ENS="${ROTDIR}/enkf${GDUMP}.${GDATE:0:8}/${GDATE:8:2}/atmos"
 
 ###############################################################
 # Run relevant script

diff --git a/jobs/JGLOBAL_ATM_ANALYSIS_INITIALIZE b/jobs/JGLOBAL_ATM_ANALYSIS_INITIALIZE
@@ -7,20 +7,31 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "atmanlinit" -c "base atmanl atmanlini
 ##############################################
 # Set variables used in the script
 ##############################################
+# shellcheck disable=SC2153
 GDATE=$(date +%Y%m%d%H -d "${PDY} ${cyc} - ${assim_freq} hours")
+gPDY=${GDATE:0:8}
+gcyc=${GDATE:8:2}
 GDUMP="gdas"
+GDUMP_ENS="enkf${GDUMP}"
 
 
 ##############################################
 # Begin JOB SPECIFIC work
 ##############################################
 
-export COMOUT=${COMOUT:-${ROTDIR}/${RUN}.${PDY}/${cyc}/atmos}
-mkdir -p "${COMOUT}"
+# Generate COM variables from templates
+YMD=${PDY} HH=${cyc} generate_com -rx COM_OBS COM_ATMOS_ANALYSIS
+
+RUN=${GDUMP} YMD=${gPDY} HH=${gcyc} generate_com -rx \
+    COM_ATMOS_ANALYSIS_PREV:COM_ATMOS_ANALYSIS_TMPL \
+    COM_ATMOS_HISTORY_PREV:COM_ATMOS_HISTORY_TMPL \
+    COM_ATMOS_RESTART_PREV:COM_ATMOS_RESTART_TMPL
+
+MEMDIR='ensstat' RUN=${GDUMP_ENS} YMD=${gPDY} HH=${gcyc} generate_com -rx \
+    COM_ATMOS_HISTORY_ENS_PREV:COM_ATMOS_HISTORY_TMPL
+
+mkdir -m 775 -p "${COM_ATMOS_ANALYSIS}"
 
-# COMIN_GES and COMIN_GES_ENS are used in script
-export COMIN_GES="${ROTDIR}/${GDUMP}.${GDATE:0:8}/${GDATE:8:2}/atmos"
-export COMIN_GES_ENS="${ROTDIR}/enkf${GDUMP}.${GDATE:0:8}/${GDATE:8:2}/atmos"
 
 ###############################################################
 # Run relevant script

diff --git a/jobs/JGLOBAL_ATM_ANALYSIS_RUN b/jobs/JGLOBAL_ATM_ANALYSIS_RUN
@@ -8,19 +8,12 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "atmanlrun" -c "base atmanl atmanlrun"
 ##############################################
 # Set variables used in the script
 ##############################################
-GDATE=$(date +%Y%m%d%H -d "${PDY} ${cyc} - ${assim_freq} hours")
-GDUMP="gdas"
+
 
 ##############################################
 # Begin JOB SPECIFIC work
 ##############################################
 
-export COMOUT=${COMOUT:-${ROTDIR}/${RUN}.${PDY}/${cyc}/atmos}
-mkdir -p "${COMOUT}"
-
-# COMIN_GES and COMIN_GES_ENS are used in script
-export COMIN_GES="${ROTDIR}/${GDUMP}.${GDATE:0:8}/${GDATE:8:2}/atmos"
-export COMIN_GES_ENS="${ROTDIR}/enkf${GDUMP}.${GDATE:0:8}/${GDATE:8:2}/atmos"
 
 ###############################################################
 # Run relevant script

diff --git a/ush/python/pygfs/task/analysis.py b/ush/python/pygfs/task/analysis.py
@@ -64,7 +64,7 @@ def get_obs_dict(self: Task) -> Dict[str, Any]:
         for ob in observers:
             obfile = ob['obs space']['obsdatain']['engine']['obsfile']
             basename = os.path.basename(obfile)
-            copylist.append([os.path.join(self.task_config['COMIN_OBS'], basename), obfile])
+            copylist.append([os.path.join(self.task_config['COM_OBS'], basename), obfile])
         obs_dict = {
             'mkdir': [os.path.join(self.runtime_config['DATA'], 'obs')],
             'copy': copylist
@@ -101,7 +101,7 @@ def get_bias_dict(self: Task) -> Dict[str, Any]:
                 prefix = '.'.join(basename.split('.')[:-2])
                 for file in ['satbias.nc4', 'satbias_cov.nc4', 'tlapse.txt']:
                     bfile = f"{prefix}.{file}"
-                    copylist.append([os.path.join(self.task_config.comin_ges_atm, bfile), os.path.join(obdir, bfile)])
+                    copylist.append([os.path.join(self.task_config.COM_ATMOS_ANALYSIS_PREV, bfile), os.path.join(obdir, bfile)])
 
         bias_dict = {
             'mkdir': [os.path.join(self.runtime_config.DATA, 'bc')],

diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py
@@ -45,7 +45,6 @@ def __init__(self, config):
                 'npz_anl': self.config.LEVS - 1,
                 'ATM_WINDOW_BEGIN': _window_begin,
                 'ATM_WINDOW_LENGTH': f"PT{self.config.assim_freq}H",
-                'comin_ges_atm': self.config.COMIN_GES,
                 'OPREFIX': f"{self.runtime_config.CDUMP}.t{self.runtime_config.cyc:02d}z.",  # TODO: CDUMP is being replaced by RUN
                 'APREFIX': f"{self.runtime_config.CDUMP}.t{self.runtime_config.cyc:02d}z.",  # TODO: CDUMP is being replaced by RUN
                 'GPREFIX': f"gdas.t{self.runtime_config.previous_cycle.hour:02d}z.",
@@ -139,7 +138,7 @@ def finalize(self: Analysis) -> None:
         """
         # ---- tar up diags
         # path of output tar statfile
-        atmstat = os.path.join(self.task_config.COMOUTatmos, f"{self.task_config.APREFIX}atmstat")
+        atmstat = os.path.join(self.task_config.COM_ATMOS_ANALYSIS, f"{self.task_config.APREFIX}atmstat")
 
         # get list of diag files to put in tarball
         diags = glob.glob(os.path.join(self.task_config.DATA, 'diags', 'diag*nc4'))
@@ -160,12 +159,12 @@ def finalize(self: Analysis) -> None:
                 archive.add(diaggzip, arcname=os.path.basename(diaggzip))
 
         # copy full YAML from executable to ROTDIR
-        logger.info(f"Copying {self.task_config.fv3jedi_yaml} to {self.task_config.COMOUTatmos}")
+        logger.info(f"Copying {self.task_config.fv3jedi_yaml} to {self.task_config.COM_ATMOS_ANALYSIS}")
         src = os.path.join(self.task_config.DATA, f"{self.task_config.CDUMP}.t{self.task_config.cyc:02d}z.atmvar.yaml")
-        dest = os.path.join(self.task_config.COMOUTatmos, f"{self.task_config.CDUMP}.t{self.task_config.cyc:02d}z.atmvar.yaml")
+        dest = os.path.join(self.task_config.COM_ATMOS_ANALYSIS, f"{self.task_config.CDUMP}.t{self.task_config.cyc:02d}z.atmvar.yaml")
         logger.debug(f"Copying {src} to {dest}")
         yaml_copy = {
-            'mkdir': [self.task_config.COMOUTatmos],
+            'mkdir': [self.task_config.COM_ATMOS_ANALYSIS],
             'copy': [[src, dest]]
         }
         FileHandler(yaml_copy).sync()
@@ -177,7 +176,7 @@ def finalize(self: Analysis) -> None:
         biaslist = []
         for bfile in biasls:
             src = os.path.join(biasdir, bfile)
-            dest = os.path.join(self.task_config.COMOUTatmos, bfile)
+            dest = os.path.join(self.task_config.COM_ATMOS_ANALYSIS, bfile)
             biaslist.append([src, dest])
 
         gprefix = f"{self.task_config.GPREFIX}"
@@ -193,11 +192,11 @@ def finalize(self: Analysis) -> None:
                 src = os.path.join(obsdir, ofile)
                 tfile = ofile.replace(gprefix, aprefix)
                 tfile = tfile.replace(gsuffix, asuffix)
-                dest = os.path.join(self.task_config.COMOUTatmos, tfile)
+                dest = os.path.join(self.task_config.COM_ATMOS_ANALYSIS, tfile)
                 biaslist.append([src, dest])
 
         bias_copy = {
-            'mkdir': [self.task_config.COMOUTatmos],
+            'mkdir': [self.task_config.COM_ATMOS_ANALYSIS],
             'copy': biaslist,
         }
         FileHandler(bias_copy).sync()
@@ -213,7 +212,7 @@ def clean(self):
     def get_bkg_dict(self, task_config: Dict[str, Any]) -> Dict[str, List[str]]:
         """Compile a dictionary of model background files to copy
 
-        This method constructs a dictionary of FV3 RESTART files (coupler, core, tracer)
+        This method constructs a dictionary of FV3 restart files (coupler, core, tracer)
         that are needed for global atm DA and returns said dictionary for use by the FileHandler class.
 
         Parameters
@@ -226,10 +225,10 @@ def get_bkg_dict(self, task_config: Dict[str, Any]) -> Dict[str, List[str]]:
         bkg_dict: Dict
             a dictionary containing the list of model background files to copy for FileHandler
         """
-        # NOTE for now this is FV3 RESTART files and just assumed to be fh006
+        # NOTE for now this is FV3 restart files and just assumed to be fh006
 
-        # get FV3 RESTART files, this will be a lot simpler when using history files
-        rst_dir = os.path.join(task_config.comin_ges_atm, 'RESTART')  # for now, option later?
+        # get FV3 restart files, this will be a lot simpler when using history files
+        rst_dir = os.path.join(task_config.COM_ATMOS_RESTART_PREV)  # for now, option later?
         run_dir = os.path.join(task_config.DATA, 'bkg')
 
         # Start accumulating list of background files to copy
@@ -415,14 +414,14 @@ def jedi2fv3inc(self: Analysis) -> None:
         case = int(self.task_config.CASE[1:])
 
         file = f"{self.task_config.GPREFIX}" + "atmf006" + f"{'' if case_anl == case else '.ensres'}" + ".nc"
-        atmges_fv3 = os.path.join(self.task_config.comin_ges_atm, file)
+        atmges_fv3 = os.path.join(self.task_config.COM_ATMOS_HISTORY_PREV, file)
 
         # Set the path/name to the input UFS-DA atm increment file (atminc_jedi)
         # and the output UFS model atm increment file (atminc_fv3)
         cdate = to_fv3time(self.task_config.current_cycle)
         cdate_inc = cdate.replace('.', '_')
         atminc_jedi = os.path.join(self.task_config.DATA, 'anl', f'atminc.{cdate_inc}z.nc4')
-        atminc_fv3 = os.path.join(self.task_config.COMOUTatmos, f"{self.task_config.CDUMP}.t{self.task_config.cyc:02d}z.atminc.nc")
+        atminc_fv3 = os.path.join(self.task_config.COM_ATMOS_ANALYSIS, f"{self.task_config.CDUMP}.t{self.task_config.cyc:02d}z.atminc.nc")
 
         # Reference the python script which does the actual work
         incpy = os.path.join(self.task_config.HOMEgfs, 'ush/jediinc2fv3.py')