Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Update UFSDA ATM anl for new COM directory structure #1537

Merged
merged 3 commits into from
Apr 26, 2023
Merged

Update UFSDA ATM anl for new COM directory structure #1537

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
60fee9e
initial refactoring of UFSDA ATM variational analysis for COM refacto…
RussTreadon-NOAA Apr 26, 2023
6ad2759
Merge branch 'NOAA-EMC:develop' into feature/ufsda_atmanl_com
RussTreadon-NOAA Apr 26, 2023
0565b4e
fix linter error and remove exports in UFSDA ATM var jobs ((#1517)
RussTreadon-NOAA Apr 26, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 11 additions & 5 deletions jobs/JGLOBAL_ATM_ANALYSIS_FINALIZE
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,19 +9,25 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "atmanlfinal" -c "base atmanl atmanlfi
# Set variables used in the script
##############################################
GDATE=$(date +%Y%m%d%H -d "${PDY} ${cyc} - ${assim_freq} hours")
export gPDY=${GDATE:0:8}
export gcyc=${GDATE:8:2}
RussTreadon-NOAA marked this conversation as resolved.
Show resolved Hide resolved
GDUMP="gdas"


##############################################
# Begin JOB SPECIFIC work
##############################################

export COMOUT=${COMOUT:-${ROTDIR}/${RUN}.${PDY}/${cyc}/atmos}
mkdir -p "${COMOUT}"
# Generate COM variables from templates
YMD=${PDY} HH=${cyc} generate_com -rx COM_ATMOS_ANALYSIS

RUN=${GDUMP} YMD=${gPDY} HH=${gcyc} generate_com -rx \
COM_ATMOS_ANALYSIS_PREV:COM_ATMOS_ANALYSIS_TMPL \
COM_ATMOS_HISTORY_PREV:COM_ATMOS_HISTORY_TMPL \
COM_ATMOS_RESTART_PREV:COM_ATMOS_RESTART_TMPL

mkdir -m 775 -p "${COM_ATMOS_ANALYSIS}"

# COMIN_GES and COMIN_GES_ENS are used in script
export COMIN_GES="${ROTDIR}/${GDUMP}.${GDATE:0:8}/${GDATE:8:2}/atmos"
export COMIN_GES_ENS="${ROTDIR}/enkf${GDUMP}.${GDATE:0:8}/${GDATE:8:2}/atmos"

###############################################################
# Run relevant script
Expand Down
20 changes: 15 additions & 5 deletions jobs/JGLOBAL_ATM_ANALYSIS_INITIALIZE
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,19 +8,29 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "atmanlinit" -c "base atmanl atmanlini
# Set variables used in the script
##############################################
GDATE=$(date +%Y%m%d%H -d "${PDY} ${cyc} - ${assim_freq} hours")
export gPDY=${GDATE:0:8}
export gcyc=${GDATE:8:2}
RussTreadon-NOAA marked this conversation as resolved.
Show resolved Hide resolved
GDUMP="gdas"
export GDUMP_ENS="enkf${GDUMP}"
RussTreadon-NOAA marked this conversation as resolved.
Show resolved Hide resolved


##############################################
# Begin JOB SPECIFIC work
##############################################

export COMOUT=${COMOUT:-${ROTDIR}/${RUN}.${PDY}/${cyc}/atmos}
mkdir -p "${COMOUT}"
# Generate COM variables from templates
YMD=${PDY} HH=${cyc} generate_com -rx COM_OBS COM_ATMOS_ANALYSIS

RUN=${GDUMP} YMD=${gPDY} HH=${gcyc} generate_com -rx \
COM_ATMOS_ANALYSIS_PREV:COM_ATMOS_ANALYSIS_TMPL \
COM_ATMOS_HISTORY_PREV:COM_ATMOS_HISTORY_TMPL \
COM_ATMOS_RESTART_PREV:COM_ATMOS_RESTART_TMPL

MEMDIR='ensstat' RUN=${GDUMP_ENS} YMD=${gPDY} HH=${gcyc} generate_com -rx \
COM_ATMOS_HISTORY_ENS_PREV:COM_ATMOS_HISTORY_TMPL

mkdir -m 775 -p "${COM_ATMOS_ANALYSIS}"

# COMIN_GES and COMIN_GES_ENS are used in script
export COMIN_GES="${ROTDIR}/${GDUMP}.${GDATE:0:8}/${GDATE:8:2}/atmos"
export COMIN_GES_ENS="${ROTDIR}/enkf${GDUMP}.${GDATE:0:8}/${GDATE:8:2}/atmos"

###############################################################
# Run relevant script
Expand Down
9 changes: 1 addition & 8 deletions jobs/JGLOBAL_ATM_ANALYSIS_RUN
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,19 +8,12 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "atmanlrun" -c "base atmanl atmanlrun"
##############################################
# Set variables used in the script
##############################################
GDATE=$(date +%Y%m%d%H -d "${PDY} ${cyc} - ${assim_freq} hours")
GDUMP="gdas"


##############################################
# Begin JOB SPECIFIC work
##############################################

export COMOUT=${COMOUT:-${ROTDIR}/${RUN}.${PDY}/${cyc}/atmos}
mkdir -p "${COMOUT}"

# COMIN_GES and COMIN_GES_ENS are used in script
export COMIN_GES="${ROTDIR}/${GDUMP}.${GDATE:0:8}/${GDATE:8:2}/atmos"
export COMIN_GES_ENS="${ROTDIR}/enkf${GDUMP}.${GDATE:0:8}/${GDATE:8:2}/atmos"

###############################################################
# Run relevant script
Expand Down
4 changes: 2 additions & 2 deletions ush/python/pygfs/task/analysis.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ def get_obs_dict(self: Task) -> Dict[str, Any]:
for ob in observers:
obfile = ob['obs space']['obsdatain']['engine']['obsfile']
basename = os.path.basename(obfile)
copylist.append([os.path.join(self.task_config['COMIN_OBS'], basename), obfile])
copylist.append([os.path.join(self.task_config['COM_OBS'], basename), obfile])
obs_dict = {
'mkdir': [os.path.join(self.runtime_config['DATA'], 'obs')],
'copy': copylist
Expand Down Expand Up @@ -101,7 +101,7 @@ def get_bias_dict(self: Task) -> Dict[str, Any]:
prefix = '.'.join(basename.split('.')[:-2])
for file in ['satbias.nc4', 'satbias_cov.nc4', 'tlapse.txt']:
bfile = f"{prefix}.{file}"
copylist.append([os.path.join(self.task_config.comin_ges_atm, bfile), os.path.join(obdir, bfile)])
copylist.append([os.path.join(self.task_config.COM_ATMOS_ANALYSIS_PREV, bfile), os.path.join(obdir, bfile)])

bias_dict = {
'mkdir': [os.path.join(self.runtime_config.DATA, 'bc')],
Expand Down
27 changes: 13 additions & 14 deletions ush/python/pygfs/task/atm_analysis.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,6 @@ def __init__(self, config):
'npz_anl': self.config.LEVS - 1,
'ATM_WINDOW_BEGIN': _window_begin,
'ATM_WINDOW_LENGTH': f"PT{self.config.assim_freq}H",
'comin_ges_atm': self.config.COMIN_GES,
'OPREFIX': f"{self.runtime_config.CDUMP}.t{self.runtime_config.cyc:02d}z.", # TODO: CDUMP is being replaced by RUN
'APREFIX': f"{self.runtime_config.CDUMP}.t{self.runtime_config.cyc:02d}z.", # TODO: CDUMP is being replaced by RUN
'GPREFIX': f"gdas.t{self.runtime_config.previous_cycle.hour:02d}z.",
Expand Down Expand Up @@ -139,7 +138,7 @@ def finalize(self: Analysis) -> None:
"""
# ---- tar up diags
# path of output tar statfile
atmstat = os.path.join(self.task_config.COMOUTatmos, f"{self.task_config.APREFIX}atmstat")
atmstat = os.path.join(self.task_config.COM_ATMOS_ANALYSIS, f"{self.task_config.APREFIX}atmstat")

# get list of diag files to put in tarball
diags = glob.glob(os.path.join(self.task_config.DATA, 'diags', 'diag*nc4'))
Expand All @@ -160,12 +159,12 @@ def finalize(self: Analysis) -> None:
archive.add(diaggzip, arcname=os.path.basename(diaggzip))

# copy full YAML from executable to ROTDIR
logger.info(f"Copying {self.task_config.fv3jedi_yaml} to {self.task_config.COMOUTatmos}")
logger.info(f"Copying {self.task_config.fv3jedi_yaml} to {self.task_config.COM_ATMOS_ANALYSIS}")
src = os.path.join(self.task_config.DATA, f"{self.task_config.CDUMP}.t{self.task_config.cyc:02d}z.atmvar.yaml")
dest = os.path.join(self.task_config.COMOUTatmos, f"{self.task_config.CDUMP}.t{self.task_config.cyc:02d}z.atmvar.yaml")
dest = os.path.join(self.task_config.COM_ATMOS_ANALYSIS, f"{self.task_config.CDUMP}.t{self.task_config.cyc:02d}z.atmvar.yaml")
logger.debug(f"Copying {src} to {dest}")
yaml_copy = {
'mkdir': [self.task_config.COMOUTatmos],
'mkdir': [self.task_config.COM_ATMOS_ANALYSIS],
'copy': [[src, dest]]
}
FileHandler(yaml_copy).sync()
Expand All @@ -177,7 +176,7 @@ def finalize(self: Analysis) -> None:
biaslist = []
for bfile in biasls:
src = os.path.join(biasdir, bfile)
dest = os.path.join(self.task_config.COMOUTatmos, bfile)
dest = os.path.join(self.task_config.COM_ATMOS_ANALYSIS, bfile)
biaslist.append([src, dest])

gprefix = f"{self.task_config.GPREFIX}"
Expand All @@ -193,11 +192,11 @@ def finalize(self: Analysis) -> None:
src = os.path.join(obsdir, ofile)
tfile = ofile.replace(gprefix, aprefix)
tfile = tfile.replace(gsuffix, asuffix)
dest = os.path.join(self.task_config.COMOUTatmos, tfile)
dest = os.path.join(self.task_config.COM_ATMOS_ANALYSIS, tfile)
biaslist.append([src, dest])

bias_copy = {
'mkdir': [self.task_config.COMOUTatmos],
'mkdir': [self.task_config.COM_ATMOS_ANALYSIS],
'copy': biaslist,
}
FileHandler(bias_copy).sync()
Expand All @@ -213,7 +212,7 @@ def clean(self):
def get_bkg_dict(self, task_config: Dict[str, Any]) -> Dict[str, List[str]]:
"""Compile a dictionary of model background files to copy

This method constructs a dictionary of FV3 RESTART files (coupler, core, tracer)
This method constructs a dictionary of FV3 restart files (coupler, core, tracer)
that are needed for global atm DA and returns said dictionary for use by the FileHandler class.

Parameters
Expand All @@ -226,10 +225,10 @@ def get_bkg_dict(self, task_config: Dict[str, Any]) -> Dict[str, List[str]]:
bkg_dict: Dict
a dictionary containing the list of model background files to copy for FileHandler
"""
# NOTE for now this is FV3 RESTART files and just assumed to be fh006
# NOTE for now this is FV3 restart files and just assumed to be fh006

# get FV3 RESTART files, this will be a lot simpler when using history files
rst_dir = os.path.join(task_config.comin_ges_atm, 'RESTART') # for now, option later?
# get FV3 restart files, this will be a lot simpler when using history files
rst_dir = os.path.join(task_config.COM_ATMOS_RESTART_PREV) # for now, option later?
run_dir = os.path.join(task_config.DATA, 'bkg')

# Start accumulating list of background files to copy
Expand Down Expand Up @@ -415,14 +414,14 @@ def jedi2fv3inc(self: Analysis) -> None:
case = int(self.task_config.CASE[1:])

file = f"{self.task_config.GPREFIX}" + "atmf006" + f"{'' if case_anl == case else '.ensres'}" + ".nc"
atmges_fv3 = os.path.join(self.task_config.comin_ges_atm, file)
atmges_fv3 = os.path.join(self.task_config.COM_ATMOS_HISTORY_PREV, file)

# Set the path/name to the input UFS-DA atm increment file (atminc_jedi)
# and the output UFS model atm increment file (atminc_fv3)
cdate = to_fv3time(self.task_config.current_cycle)
cdate_inc = cdate.replace('.', '_')
atminc_jedi = os.path.join(self.task_config.DATA, 'anl', f'atminc.{cdate_inc}z.nc4')
atminc_fv3 = os.path.join(self.task_config.COMOUTatmos, f"{self.task_config.CDUMP}.t{self.task_config.cyc:02d}z.atminc.nc")
atminc_fv3 = os.path.join(self.task_config.COM_ATMOS_ANALYSIS, f"{self.task_config.CDUMP}.t{self.task_config.cyc:02d}z.atminc.nc")

# Reference the python script which does the actual work
incpy = os.path.join(self.task_config.HOMEgfs, 'ush/jediinc2fv3.py')
Expand Down