reverted stage.atmos stuff

ush/ufsda/stage.py

@@ -12,7 +12,7 @@
 import logging
 import glob
 import numpy as np
-from pygw.yaml_file import YAMLFile, parse_yaml
+from pygw.yaml_file import YAMLFile, parse_yaml, parse_j2yaml
 import ufsda.soca_utils
 
 __all__ = ['atm_background', 'atm_obs', 'bias_obs', 'background', 'background_ens', 'fv3jedi', 'obs', 'berror', 'gdas_fix', 'gdas_single_cycle']
@@ -117,6 +117,125 @@ def soca_fix(config):
                               os.path.join(config['stage_dir'], 'INPUT'))
 
 
+def atm_background(config):
+    # stage FV3 backgrounds
+    r2d2_config = {
+        'start': config['prev_valid_time'],
+        'end': config['prev_valid_time'],
+        'step': config['atm_window_length'],
+        'forecast_steps': ['PT6H'],  # 3DVar no FGAT for now
+        'file_type_list': ['fv_core.res', 'fv_srf_wnd.res', 'fv_tracer.res', 'phy_data', 'sfc_data'],
+        'target_dir': config.get('target_dir', config.get('BKG_DIR', './')),
+        'target_file_fmt': '{target_dir}/$(valid_date).$(file_type).tile$(tile).nc',
+        'type': 'fc',
+        'model': 'gfs',
+        'resolution': config['CASE'].lower(),
+        'dump': 'gdas',
+        'experiment': 'oper_gdas',  # change this here and other places to be oper_{dump}
+        'tile': [1, 2, 3, 4, 5, 6],
+        'user_date_format': '%Y%m%d.%H%M%S',
+        'fc_date_rendering': 'analysis',
+    }
+    r2d2_config = NiceDict(r2d2_config)
+    ufsda.r2d2.fetch(r2d2_config)
+    # get gfs metadata
+    r2d2_config['model'] = 'gfs_metadata'
+    r2d2_config['target_file_fmt'] = '{target_dir}/$(valid_date).$(file_type)'
+    r2d2_config['file_type_list'] = ['coupler.res', 'fv_core.res.nc']
+    del r2d2_config['tile']
+    ufsda.r2d2.fetch(r2d2_config)
+
+
+def atm_obs(config, local_dict):
+    # fetch atm analysis obs
+    r2d2_config = {
+        'start': config['prev_valid_time'],
+        'end': config['prev_valid_time'],
+        'step': config['ATM_WINDOW_LENGTH'],
+        'dump': 'gdas',
+        'experiment': 'oper_gdas',  # change this here and other places to be oper_{dump}
+        'target_dir': config.get('target_dir', os.path.join(config.get('DATA', './'), 'obs')),
+    }
+    r2d2_config = NiceDict(r2d2_config)
+    # get list of obs to process and their output files
+    obs_list_yaml = config['OBS_LIST']
+    obs_list_config = Configuration(obs_list_yaml)
+    obs_list_config = ufsda.yamltools.iter_config(config, obs_list_config)
+
+    for ob in obs_list_config['observers']:
+        ob_config = parse_j2yaml(ob, local_dict)
+        # first get obs
+        r2d2_config.pop('file_type', None)
+        r2d2_config['type'] = 'ob'
+        r2d2_config['provider'] = 'ncdiag'
+        r2d2_config['start'] = config['ATM_WINDOW_BEGIN']
+        r2d2_config['end'] = r2d2_config['start']
+        ob_basename = os.path.basename(ob_config['obs space']['obsdatain']['engine']['obsfile'])
+        target_file = os.path.join(os.environ['COMOUT'], ob_basename)
+        r2d2_config['target_file_fmt'] = target_file
+        r2d2_config['obs_types'] = [ob_config['obs space']['name']]
+        ufsda.r2d2.fetch(r2d2_config)
+
+
+def bias_obs(config, local_dict):
+    # fetch bias files
+    r2d2_config = {
+        'start': config['prev_valid_time'],
+        'end': config['prev_valid_time'],
+        'step': config['atm_window_length'],
+        'dump': 'gdas',
+        'experiment': 'oper_gdas',  # change this here and other places to be oper_{dump}
+        'target_dir': config.get('target_dir', config.get('COMIN_GES', './')),
+    }
+    r2d2_config = NiceDict(r2d2_config)
+    # get list of obs to process and their output files
+    obs_list_yaml = config['OBS_LIST']
+    obs_list_config = Configuration(obs_list_yaml)
+    obs_list_config = ufsda.yamltools.iter_config(config, obs_list_config)
+
+    for ob in obs_list_config['observers']:
+        ob_config = parse_j2yaml(ob, local_dict)
+        r2d2_config.pop('file_type', None)
+        r2d2_config['obs_types'] = [ob_config['obs space']['name']]
+        # get bias files if needed
+        if 'obs bias' in ob_config.keys():
+            r2d2_config['type'] = 'bc'
+            r2d2_config['provider'] = 'gsi'
+            r2d2_config['start'] = config['prev_valid_time']
+            r2d2_config['end'] = r2d2_config['start']
+
+            # fetch satbias
+            r2d2_config['file_type'] = 'satbias'
+            target_file = ob_config['obs bias']['input file']
+            ob_basename = os.path.basename(target_file)
+            target_file = os.path.join(os.environ['COMOUT'], ob_basename)
+            r2d2_config['target_file_fmt'] = target_file
+            r2d2_config['experiment'] = config.get('experiment', 'oper_gdas')
+            ufsda.r2d2.fetch(r2d2_config)
+
+            # fetch satbias_cov    # note:  non-standard R2D2 added for cycling
+            r2d2_config['file_type'] = 'satbias_cov'
+            target_file2 = target_file.replace('satbias', 'satbias_cov')
+            r2d2_config['target_file_fmt'] = target_file2
+            r2d2_config['experiment'] = config.get('experiment', 'oper_gdas')
+            try:
+                ufsda.r2d2.fetch(r2d2_config)
+            except FileNotFoundError:
+                logging.error("Warning: satbias_cov file cannot be fetched from R2D2!")
+                # temp hack to copy satbias as satbias_cov
+                # if satbias_cov does not exists in R2D2
+                if os.path.isfile(target_file) and not os.path.isfile(target_file2):
+                    shutil.copy(target_file, target_file2)
+
+            # fetch tlapse
+            r2d2_config['file_type'] = 'tlapse'
+            target_file = target_file.replace('satbias', 'tlapse')
+            target_file = target_file.replace('nc4', 'txt')
+            r2d2_config['target_file_fmt'] = target_file
+            r2d2_config['experiment'] = 'oper_gdas'
+            ufsda.r2d2.fetch(r2d2_config)
+
+
 def gdas_single_cycle(config, local_dict):
     # grab backgrounds first
     r2d2_config = {