Skip to content

Commit

Permalink
Update aerosol DA to use new COM structure (#1551)
Browse files Browse the repository at this point in the history 
This PR updates the j-jobs and python classes for aerosol DA to use the new COM directory structure.
This PR also includes removing of the chem history staging for the ICSDIR in setup_expt.py. The aerosol fields are treated as FV3 tracers, so they are either available (warm start) with the fv_tracer files or start at 0 (cold start) and must be spun up from emissions.

Fixes #1516
  • Loading branch information
@CoryMartin-NOAA
CoryMartin-NOAA committed May 1, 2023
1 parent ec2dd3a commit 6c48e94
Show file tree
Hide file tree
Showing 6 changed files with 30 additions and 43 deletions.
15 changes: 10 additions & 5 deletions jobs/JGLOBAL_AERO_ANALYSIS_FINALIZE
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,20 +8,25 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "aeroanlfinal" -c "base aeroanl aeroan
##############################################
# Set variables used in the script
##############################################
# shellcheck disable=SC2153
GDATE=$(date +%Y%m%d%H -d "${PDY} ${cyc} - ${assim_freq} hours")
gPDY=${GDATE:0:8}
gcyc=${GDATE:8:2}
GDUMP="gdas"


##############################################
# Begin JOB SPECIFIC work
##############################################

export COMOUT=${COMOUT:-${ROTDIR}/${RUN}.${PDY}/${cyc}/chem}
mkdir -p "${COMOUT}"
# Generate COM variables from templates
YMD=${PDY} HH=${cyc} generate_com -rx COM_OBS COM_CHEM_ANALYSIS

# COMIN_GES and COMIN_GES_ENS are used in script
export COMIN_GES="${ROTDIR}/${GDUMP}.${GDATE:0:8}/${GDATE:8:2}/chem"
export COMIN_GES_ENS="${ROTDIR}/enkf${GDUMP}.${GDATE:0:8}/${GDATE:8:2}/chem"
RUN=${GDUMP} YMD=${gPDY} HH=${gcyc} generate_com -rx \
COM_CHEM_ANALYSIS_PREV:COM_CHEM_ANALYSIS_TMPL \
COM_ATMOS_RESTART_PREV:COM_ATMOS_RESTART_TMPL

mkdir -m 775 -p "${COM_CHEM_ANALYSIS}"

###############################################################
# Run relevant script
Expand Down
15 changes: 10 additions & 5 deletions jobs/JGLOBAL_AERO_ANALYSIS_INITIALIZE
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,20 +7,25 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "aeroanlinit" -c "base aeroanl aeroanl
##############################################
# Set variables used in the script
##############################################
# shellcheck disable=SC2153
GDATE=$(date +%Y%m%d%H -d "${PDY} ${cyc} - ${assim_freq} hours")
gPDY=${GDATE:0:8}
gcyc=${GDATE:8:2}
GDUMP="gdas"


##############################################
# Begin JOB SPECIFIC work
##############################################

export COMOUT=${COMOUT:-${ROTDIR}/${RUN}.${PDY}/${cyc}/chem}
mkdir -p "${COMOUT}"
# Generate COM variables from templates
YMD=${PDY} HH=${cyc} generate_com -rx COM_OBS COM_CHEM_ANALYSIS

# COMIN_GES and COMIN_GES_ENS are used in script
export COMIN_GES="${ROTDIR}/${GDUMP}.${GDATE:0:8}/${GDATE:8:2}/chem"
export COMIN_GES_ENS="${ROTDIR}/enkf${GDUMP}.${GDATE:0:8}/${GDATE:8:2}/chem"
RUN=${GDUMP} YMD=${gPDY} HH=${gcyc} generate_com -rx \
COM_CHEM_ANALYSIS_PREV:COM_CHEM_ANALYSIS_TMPL \
COM_ATMOS_RESTART_PREV:COM_ATMOS_RESTART_TMPL

mkdir -m 775 -p "${COM_CHEM_ANALYSIS}"

###############################################################
# Run relevant script
Expand Down
9 changes: 0 additions & 9 deletions jobs/JGLOBAL_AERO_ANALYSIS_RUN
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,20 +8,11 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "aeroanlrun" -c "base aeroanl aeroanlr
##############################################
# Set variables used in the script
##############################################
GDATE=$(date +%Y%m%d%H -d "${PDY} ${cyc} - ${assim_freq} hours")
GDUMP="gdas"

##############################################
# Begin JOB SPECIFIC work
##############################################

export COMOUT=${COMOUT:-${ROTDIR}/${RUN}.${PDY}/${cyc}/chem}
mkdir -p "${COMOUT}"

# COMIN_GES and COMIN_GES_ENS are used in script
export COMIN_GES="${ROTDIR}/${GDUMP}.${GDATE:0:8}/${GDATE:8:2}/chem"
export COMIN_GES_ENS="${ROTDIR}/enkf${GDUMP}.${GDATE:0:8}/${GDATE:8:2}/chem"

###############################################################
# Run relevant script

Expand Down
1 change: 1 addition & 0 deletions parm/config/config.com
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,5 +87,6 @@ declare -rx COM_ICE_HISTORY_TMPL=${COM_BASE}'/model_data/ice/history'
declare -rx COM_ICE_RESTART_TMPL=${COM_BASE}'/model_data/ice/restart'

declare -rx COM_CHEM_HISTORY_TMPL=${COM_BASE}'/model_data/chem/history'
declare -rx COM_CHEM_ANALYSIS_TMPL=${COM_BASE}'/analysis/chem'

declare -rx COM_MED_RESTART_TMPL=${COM_BASE}'/model_data/med/restart'
17 changes: 8 additions & 9 deletions ush/python/pygfs/task/aero_analysis.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,6 @@ def __init__(self, config):
'npz_anl': self.config['LEVS'] - 1,
'AERO_WINDOW_BEGIN': _window_begin,
'AERO_WINDOW_LENGTH': f"PT{self.config['assim_freq']}H",
'comin_ges_atm': self.config.COMIN_GES.replace('chem', 'atmos'), # 'chem' is COMPONENT, aerosol fields are in 'atmos' tracers
'OPREFIX': f"{self.runtime_config.CDUMP}.t{self.runtime_config.cyc:02d}z.", # TODO: CDUMP is being replaced by RUN
'APREFIX': f"{self.runtime_config.CDUMP}.t{self.runtime_config.cyc:02d}z.", # TODO: CDUMP is being replaced by RUN
'GPREFIX': f"gdas.t{self.runtime_config.previous_cycle.hour:02d}z.",
Expand Down Expand Up @@ -143,7 +142,7 @@ def finalize(self: Analysis) -> None:
"""
# ---- tar up diags
# path of output tar statfile
aerostat = os.path.join(self.task_config['COMOUTaero'], f"{self.task_config['APREFIX']}aerostat")
aerostat = os.path.join(self.task_config.COM_CHEM_ANALYSIS, f"{self.task_config['APREFIX']}aerostat")

# get list of diag files to put in tarball
diags = glob.glob(os.path.join(self.task_config['DATA'], 'diags', 'diag*nc4'))
Expand All @@ -161,9 +160,9 @@ def finalize(self: Analysis) -> None:

# copy full YAML from executable to ROTDIR
src = os.path.join(self.task_config['DATA'], f"{self.task_config['CDUMP']}.t{self.runtime_config['cyc']:02d}z.aerovar.yaml")
dest = os.path.join(self.task_config['COMOUTaero'], f"{self.task_config['CDUMP']}.t{self.runtime_config['cyc']:02d}z.aerovar.yaml")
dest = os.path.join(self.task_config.COM_CHEM_ANALYSIS, f"{self.task_config['CDUMP']}.t{self.runtime_config['cyc']:02d}z.aerovar.yaml")
yaml_copy = {
'mkdir': [self.task_config['COMOUTaero']],
'mkdir': [self.task_config.COM_CHEM_ANALYSIS],
'copy': [[src, dest]]
}
FileHandler(yaml_copy).sync()
Expand All @@ -175,8 +174,8 @@ def finalize(self: Analysis) -> None:
bkglist = []
for itile in range(1, self.task_config.ntiles + 1):
tracer = template.format(tilenum=itile)
src = os.path.join(self.task_config.comin_ges_atm, 'RESTART', tracer)
dest = os.path.join(self.task_config.COMOUTaero, f'aeroges.{tracer}')
src = os.path.join(self.task_config.COM_ATMOS_RESTART_PREV, tracer)
dest = os.path.join(self.task_config.COM_CHEM_ANALYSIS, f'aeroges.{tracer}')
bkglist.append([src, dest])
FileHandler({'copy': bkglist}).sync()

Expand All @@ -191,7 +190,7 @@ def finalize(self: Analysis) -> None:
for itile in range(1, self.task_config.ntiles + 1):
tracer = template.format(tilenum=itile)
src = os.path.join(self.task_config.DATA, 'anl', tracer)
dest = os.path.join(self.task_config.COMOUTaero, tracer)
dest = os.path.join(self.task_config.COM_CHEM_ANALYSIS, tracer)
inclist.append([src, dest])
FileHandler({'copy': inclist}).sync()

Expand All @@ -207,7 +206,7 @@ def _add_fms_cube_sphere_increments(self: Analysis) -> None:
# only need the fv_tracer files
template = f'{to_fv3time(self.task_config.current_cycle)}.fv_tracer.res.tile{{tilenum}}.nc'
inc_template = os.path.join(self.task_config.DATA, 'anl', 'aeroinc.' + template)
bkg_template = os.path.join(self.task_config.comin_ges_atm, 'RESTART', template)
bkg_template = os.path.join(self.task_config.COM_ATMOS_RESTART_PREV, template)
# get list of increment vars
incvars_list_path = os.path.join(self.task_config['HOMEgfs'], 'parm', 'parm_gdas', 'aeroanl_inc_vars.yaml')
incvars = YAMLFile(path=incvars_list_path)['incvars']
Expand All @@ -233,7 +232,7 @@ def get_bkg_dict(self, task_config: Dict[str, Any]) -> Dict[str, List[str]]:
# NOTE for now this is FV3 RESTART files and just assumed to be fh006

# get FV3 RESTART files, this will be a lot simpler when using history files
rst_dir = os.path.join(task_config.comin_ges_atm, 'RESTART') # for now, option later?
rst_dir = task_config.COM_ATMOS_RESTART_PREV
run_dir = os.path.join(task_config['DATA'], 'bkg')

# Start accumulating list of background files to copy
Expand Down
16 changes: 1 addition & 15 deletions workflow/setup_expt.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,14 +58,11 @@ def fill_COMROT_cycled(host, inputs):

comrot = os.path.join(inputs.comrot, inputs.pslot)

do_ocean = do_ice = do_med = do_aerosols = False
do_ocean = do_ice = do_med = False

if inputs.app in ['S2S', 'S2SW']:
do_ocean = do_ice = do_med = True

if inputs.app in ['ATMA']:
do_aerosols = True

if inputs.icsdir is None:
warnings.warn("User did not provide '--icsdir' to stage initial conditions")
return
Expand Down Expand Up @@ -182,17 +179,6 @@ def link_files_from_src_to_dst(src_dir, dst_dir):
makedirs_if_missing(dst_dir)
link_files_from_src_to_dst(src_dir, dst_dir)

# Link aerosol files
if do_aerosols:
if inputs.start in ['warm']:
detdir = f'{inputs.cdump}.{rdatestr[:8]}/{rdatestr[8:]}'
elif inputs.start in ['cold']:
detdir = f'{inputs.cdump}.{idatestr[:8]}/{idatestr[8:]}'
dst_dir = os.path.join(comrot, detdir, chem_dir)
src_dir = os.path.join(inputs.icsdir, detdir, chem_dir)
makedirs_if_missing(dst_dir)
link_files_from_src_to_dst(src_dir, dst_dir)

# Link bias correction and radiance diagnostics files
detdir = f'{inputs.cdump}.{idatestr[:8]}/{idatestr[8:]}'
src_dir = os.path.join(inputs.icsdir, detdir, src_atm_anl_dir)
Expand Down

0 comments on commit 6c48e94

Please sign in to comment.