Skip to content

Commit

Permalink
Merge branch 'develop' into feature/ss160
Browse files Browse the repository at this point in the history
  • Loading branch information
@CoryMartin-NOAA
CoryMartin-NOAA committed Jan 25, 2024
2 parents 92a26b4 + ac8bd9c commit fafeddb
Show file tree
Hide file tree
Showing 4 changed files with 163 additions and 206 deletions.
9 changes: 4 additions & 5 deletions scripts/exgdas_global_marine_analysis_post.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,10 @@
import os
import glob
import shutil
import logging
from datetime import datetime, timedelta
from wxflow import FileHandler
from wxflow import FileHandler, Logger

logger = Logger()


# TODO: Move this somewhere else?
Expand All @@ -35,9 +36,7 @@ def list_all_files(dir_in, dir_out, wc='*', fh_list=[]):
return fh_list


# set up logger
logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s', level=logging.INFO, datefmt='%Y-%m-%d %H:%M:%S')
logging.info(f"---------------- Copy from RUNDIR to COMOUT")
logger.info(f"---------------- Copy from RUNDIR to COMOUT")

com_ocean_analysis = os.getenv('COM_OCEAN_ANALYSIS')
com_ice_restart = os.getenv('COM_ICE_RESTART')
Expand Down
58 changes: 28 additions & 30 deletions scripts/exgdas_global_marine_analysis_prep.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,17 +26,15 @@
import dateutil.parser as dparser
import f90nml
import shutil
import logging
import subprocess
from datetime import datetime, timedelta
import pytz
from netCDF4 import Dataset
import xarray as xr
import numpy as np
from wxflow import (AttrDict, Template, TemplateConstants, YAMLFile, FileHandler)
from wxflow import (AttrDict, Logger, Template, TemplateConstants, YAMLFile, FileHandler)

# set up logger
logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s', level=logging.INFO, datefmt='%Y-%m-%d %H:%M:%S')
logger = Logger()

# get absolute path of ush/ directory either from env or relative to this file
my_dir = os.path.dirname(__file__)
Expand Down Expand Up @@ -91,7 +89,7 @@ def cice_hist2fms(input_filename, output_filename):
ds = xr.open_dataset(input_filename_real)

if 'aicen' in ds.variables and 'hicen' in ds.variables and 'hsnon' in ds.variables:
logging.info(f"*** Already reformatted, skipping.")
logger.info(f"*** Already reformatted, skipping.")
return

# rename the dimensions to xaxis_1 and yaxis_1
Expand All @@ -114,7 +112,7 @@ def test_hist_date(histfile, ref_date):
ncf = Dataset(histfile, 'r')
hist_date = dparser.parse(ncf.variables['time'].units, fuzzy=True) + timedelta(hours=int(ncf.variables['time'][0]))
ncf.close()
logging.info(f"*** history file date: {hist_date} expected date: {ref_date}")
logger.info(f"*** history file date: {hist_date} expected date: {ref_date}")
assert hist_date == ref_date, 'Inconsistent bkg date'


Expand Down Expand Up @@ -225,7 +223,7 @@ def find_clim_ens(input_date):
################################################################################
# runtime environment variables, create directories

logging.info(f"---------------- Setup runtime environement")
logger.info(f"---------------- Setup runtime environement")

comin_obs = os.getenv('COMIN_OBS')
anl_dir = os.getenv('DATA')
Expand Down Expand Up @@ -259,7 +257,7 @@ def find_clim_ens(input_date):
################################################################################
# fetch observations

logging.info(f"---------------- Stage observations")
logger.info(f"---------------- Stage observations")

# create config dict from runtime env
envconfig = {'window_begin': f"{window_begin.strftime('%Y-%m-%dT%H:%M:%SZ')}",
Expand All @@ -278,29 +276,29 @@ def find_clim_ens(input_date):

# copy obs from COM_OBS to DATA/obs
for obs_file in obs_files:
logging.info(f"******* {obs_file}")
logger.info(f"******* {obs_file}")
obs_src = os.path.join(os.getenv('COM_OBS'), obs_file)
obs_dst = os.path.join(os.path.realpath(obs_in), obs_file)
logging.info(f"******* {obs_src}")
logger.info(f"******* {obs_src}")
if os.path.exists(obs_src):
logging.info(f"******* fetching {obs_file}")
logger.info(f"******* fetching {obs_file}")
obs_list.append([obs_src, obs_dst])
else:
logging.info(f"******* {obs_file} is not in the database")
logger.info(f"******* {obs_file} is not in the database")

FileHandler({'copy': obs_list}).sync()

################################################################################
# stage static files

logging.info(f"---------------- Stage static files")
logger.info(f"---------------- Stage static files")
ufsda.stage.soca_fix(stage_cfg)


################################################################################
# stage ensemble members
if dohybvar:
logging.info("---------------- Stage ensemble members")
logger.info("---------------- Stage ensemble members")
nmem_ens = int(os.getenv('NMEM_ENS'))
longname = {'ocn': 'ocean', 'ice': 'ice'}
ens_member_list = []
Expand All @@ -324,7 +322,7 @@ def find_clim_ens(input_date):
cice_fname = os.path.realpath(os.path.join(static_ens, "ice."+str(mem)+".nc"))
cice_hist2fms(cice_fname, cice_fname)
else:
logging.info("---------------- Stage offline ensemble members")
logger.info("---------------- Stage offline ensemble members")
ens_member_list = []
clim_ens_dir = find_clim_ens(pytz.utc.localize(window_begin, is_dst=None))
nmem_ens = len(glob.glob(os.path.abspath(os.path.join(clim_ens_dir, 'ocn.*.nc'))))
Expand All @@ -340,12 +338,12 @@ def find_clim_ens(input_date):
################################################################################
# prepare JEDI yamls

logging.info(f"---------------- Generate JEDI yaml files")
logger.info(f"---------------- Generate JEDI yaml files")

################################################################################
# copy yaml for grid generation

logging.info(f"---------------- generate gridgen.yaml")
logger.info(f"---------------- generate gridgen.yaml")
gridgen_yaml_src = os.path.realpath(os.path.join(gdas_home, 'parm', 'soca', 'gridgen', 'gridgen.yaml'))
gridgen_yaml_dst = os.path.realpath(os.path.join(stage_cfg['stage_dir'], 'gridgen.yaml'))
FileHandler({'copy': [[gridgen_yaml_src, gridgen_yaml_dst]]}).sync()
Expand All @@ -354,14 +352,14 @@ def find_clim_ens(input_date):
# generate the YAML file for the post processing of the clim. ens. B
berror_yaml_dir = os.path.join(gdas_home, 'parm', 'soca', 'berror')

logging.info(f"---------------- generate soca_ensb.yaml")
logger.info(f"---------------- generate soca_ensb.yaml")
berr_yaml = os.path.join(anl_dir, 'soca_ensb.yaml')
berr_yaml_template = os.path.join(berror_yaml_dir, 'soca_ensb.yaml')
config = YAMLFile(path=berr_yaml_template)
config = Template.substitute_structure(config, TemplateConstants.DOUBLE_CURLY_BRACES, envconfig.get)
config.save(berr_yaml)

logging.info(f"---------------- generate soca_ensweights.yaml")
logger.info(f"---------------- generate soca_ensweights.yaml")
berr_yaml = os.path.join(anl_dir, 'soca_ensweights.yaml')
berr_yaml_template = os.path.join(berror_yaml_dir, 'soca_ensweights.yaml')
config = YAMLFile(path=berr_yaml_template)
Expand All @@ -371,31 +369,31 @@ def find_clim_ens(input_date):
################################################################################
# copy yaml for localization length scales

logging.info(f"---------------- generate soca_setlocscales.yaml")
logger.info(f"---------------- generate soca_setlocscales.yaml")
locscales_yaml_src = os.path.join(gdas_home, 'parm', 'soca', 'berror', 'soca_setlocscales.yaml')
locscales_yaml_dst = os.path.join(stage_cfg['stage_dir'], 'soca_setlocscales.yaml')
FileHandler({'copy': [[locscales_yaml_src, locscales_yaml_dst]]}).sync()

################################################################################
# copy yaml for correlation length scales

logging.info(f"---------------- generate soca_setcorscales.yaml")
logger.info(f"---------------- generate soca_setcorscales.yaml")
corscales_yaml_src = os.path.join(gdas_home, 'parm', 'soca', 'berror', 'soca_setcorscales.yaml')
corscales_yaml_dst = os.path.join(stage_cfg['stage_dir'], 'soca_setcorscales.yaml')
FileHandler({'copy': [[corscales_yaml_src, corscales_yaml_dst]]}).sync()

################################################################################
# copy yaml for diffusion initialization

logging.info(f"---------------- generate soca_parameters_diffusion_hz.yaml")
logger.info(f"---------------- generate soca_parameters_diffusion_hz.yaml")
diffu_hz_yaml = os.path.join(anl_dir, 'soca_parameters_diffusion_hz.yaml')
diffu_hz_yaml_dir = os.path.join(gdas_home, 'parm', 'soca', 'berror')
diffu_hz_yaml_template = os.path.join(berror_yaml_dir, 'soca_parameters_diffusion_hz.yaml')
config = YAMLFile(path=diffu_hz_yaml_template)
config = Template.substitute_structure(config, TemplateConstants.DOUBLE_CURLY_BRACES, envconfig.get)
config.save(diffu_hz_yaml)

logging.info(f"---------------- generate soca_parameters_diffusion_vt.yaml")
logger.info(f"---------------- generate soca_parameters_diffusion_vt.yaml")
diffu_vt_yaml = os.path.join(anl_dir, 'soca_parameters_diffusion_vt.yaml')
diffu_vt_yaml_dir = os.path.join(gdas_home, 'parm', 'soca', 'berror')
diffu_vt_yaml_template = os.path.join(berror_yaml_dir, 'soca_parameters_diffusion_vt.yaml')
Expand All @@ -406,7 +404,7 @@ def find_clim_ens(input_date):
################################################################################
# generate yaml for bump/nicas (used for correlation and/or localization)

logging.info(f"---------------- generate BUMP/NICAS localization yamls")
logger.info(f"---------------- generate BUMP/NICAS localization yamls")
# localization bump yaml
bumpdir = 'bump'
ufsda.disk_utils.mkdir(os.path.join(anl_dir, bumpdir))
Expand All @@ -424,7 +422,7 @@ def find_clim_ens(input_date):
################################################################################
# generate yaml for soca_var

logging.info(f"---------------- generate var.yaml")
logger.info(f"---------------- generate var.yaml")
var_yaml = os.path.join(anl_dir, 'var.yaml')
var_yaml_template = os.path.join(gdas_home,
'parm',
Expand All @@ -440,13 +438,13 @@ def find_clim_ens(input_date):
# select the SABER BLOCKS to use
if 'SABER_BLOCKS_YAML' in os.environ and os.environ['SABER_BLOCKS_YAML']:
saber_blocks_yaml = os.getenv('SABER_BLOCKS_YAML')
logging.info(f"using non-default SABER blocks yaml: {saber_blocks_yaml}")
logger.info(f"using non-default SABER blocks yaml: {saber_blocks_yaml}")
else:
logging.info(f"using default SABER blocks yaml")
logger.info(f"using default SABER blocks yaml")
os.environ['SABER_BLOCKS_YAML'] = os.path.join(gdas_home, 'parm', 'soca', 'berror', 'saber_blocks.yaml')

# substitute templated variables in the var config
logging.info(f"{config}")
logger.info(f"{config}")
varconfig = YAMLFile(path=var_yaml_template)
varconfig = Template.substitute_structure(varconfig, TemplateConstants.DOUBLE_CURLY_BRACES, config.get)
varconfig = Template.substitute_structure(varconfig, TemplateConstants.DOLLAR_PARENTHESES, config.get)
Expand All @@ -460,7 +458,7 @@ def find_clim_ens(input_date):
# Prepare the yamls for the "checkpoint" jjob
# prepare yaml and CICE restart for soca to cice change of variable

logging.info(f"---------------- generate soca to cice yamls")
logger.info(f"---------------- generate soca to cice yamls")
# make a copy of the CICE6 restart
rst_date = fcst_begin.strftime('%Y%m%d.%H%M%S')
ice_rst = os.path.join(os.getenv('COM_ICE_RESTART_PREV'), f'{rst_date}.cice_model.res.nc')
Expand All @@ -483,7 +481,7 @@ def find_clim_ens(input_date):
outyaml.save(varchgyaml)

# prepare yaml for soca to MOM6 IAU increment
logging.info(f"---------------- generate soca to MOM6 IAU yaml")
logger.info(f"---------------- generate soca to MOM6 IAU yaml")
socaincr2mom6_yaml = os.path.join(anl_dir, 'socaincr2mom6.yaml')
socaincr2mom6_yaml_template = os.path.join(gdas_home, 'parm', 'soca', 'variational', 'socaincr2mom6.yaml')
s2mconfig = YAMLFile(path=socaincr2mom6_yaml_template)
Expand Down
Loading

0 comments on commit fafeddb

Please sign in to comment.