Merge branch 'develop' into feature/fix-versioning

* develop: Fix preamble id (#996) Add missing "atmos" into job dependencies (#998) Bugfix in arch.sh to remove hardwired "htar" (#992) Add in stubs for aerosol DA tasks + bugfix for setup_expt where cycled and ATMA are used (#990)
NOAA-EMC · Aug 25, 2022 · 3711ded · 3711ded
2 parents cbdea98 + 1f14229
commit 3711ded
Show file tree

Hide file tree

Showing 15 changed files with 117 additions and 16 deletions.
diff --git a/jobs/rocoto/aeroanlfinal.sh b/jobs/rocoto/aeroanlfinal.sh
@@ -0,0 +1,12 @@
+#! /usr/bin/env bash
+
+source "$HOMEgfs/ush/preamble.sh"
+
+###############################################################
+# Source FV3GFS workflow modules
+. $HOMEgfs/ush/load_fv3gfs_modules.sh
+status=$?
+[[ $status -ne 0 ]] && exit $status
+
+###############################################################
+echo "Do nothing for now"
diff --git a/jobs/rocoto/aeroanlinit.sh b/jobs/rocoto/aeroanlinit.sh
@@ -0,0 +1,12 @@
+#! /usr/bin/env bash
+
+source "$HOMEgfs/ush/preamble.sh"
+
+###############################################################
+# Source FV3GFS workflow modules
+. $HOMEgfs/ush/load_fv3gfs_modules.sh
+status=$?
+[[ $status -ne 0 ]] && exit $status
+
+###############################################################
+echo "Do nothing for now"
diff --git a/jobs/rocoto/aeroanlrun.sh b/jobs/rocoto/aeroanlrun.sh
@@ -0,0 +1,12 @@
+#! /usr/bin/env bash
+
+source "$HOMEgfs/ush/preamble.sh"
+
+###############################################################
+# Source FV3GFS workflow modules
+. $HOMEgfs/ush/load_fv3gfs_modules.sh
+status=$?
+[[ $status -ne 0 ]] && exit $status
+
+###############################################################
+echo "Do nothing for now"
diff --git a/jobs/rocoto/arch.sh b/jobs/rocoto/arch.sh
@@ -213,7 +213,7 @@ if [ $CDUMP = "gfs" ]; then
     # Aerosols
     if [ $DO_AERO = "YES" ]; then
         for targrp in chem; do
-            htar -P -cvf $ATARDIR/$CDATE/${targrp}.tar $(cat $ARCH_LIST/${targrp}.txt)
+            $TARCMD -P -cvf $ATARDIR/$CDATE/${targrp}.tar $(cat $ARCH_LIST/${targrp}.txt)
             status=$?
             if [ $status -ne 0 -a $CDATE -ge $firstday ]; then
                 echo "HTAR $CDATE ${targrp}.tar failed"

diff --git a/jobs/rocoto/atmanalpost.sh b/jobs/rocoto/atmanalpost.sh
@@ -1,4 +1,6 @@
-#!/bin/bash -x
+#! /usr/bin/env bash
+
+source "$HOMEgfs/ush/preamble.sh"
 
 ###############################################################
 # Source FV3GFS workflow modules

diff --git a/jobs/rocoto/atmanalprep.sh b/jobs/rocoto/atmanalprep.sh
@@ -1,4 +1,6 @@
-#!/bin/bash -x
+#! /usr/bin/env bash
+
+source "$HOMEgfs/ush/preamble.sh"
 
 ###############################################################
 # Source FV3GFS workflow modules

diff --git a/jobs/rocoto/atmanalrun.sh b/jobs/rocoto/atmanalrun.sh
@@ -1,4 +1,6 @@
-#!/bin/bash -x
+#! /usr/bin/env bash
+
+source "$HOMEgfs/ush/preamble.sh"
 
 ###############################################################
 # Source FV3GFS workflow modules

diff --git a/jobs/rocoto/atmensanalpost.sh b/jobs/rocoto/atmensanalpost.sh
@@ -1,4 +1,6 @@
-#!/bin/bash -x
+#! /usr/bin/env bash
+
+source "$HOMEgfs/ush/preamble.sh"
 
 ###############################################################
 # Source FV3GFS workflow modules

diff --git a/jobs/rocoto/atmensanalprep.sh b/jobs/rocoto/atmensanalprep.sh
@@ -1,4 +1,6 @@
-#!/bin/bash -x
+#! /usr/bin/env bash
+
+source "$HOMEgfs/ush/preamble.sh"
 
 ###############################################################
 # Source FV3GFS workflow modules

diff --git a/jobs/rocoto/atmensanalrun.sh b/jobs/rocoto/atmensanalrun.sh
@@ -1,4 +1,6 @@
-#!/bin/bash -x
+#! /usr/bin/env bash
+
+source "$HOMEgfs/ush/preamble.sh"
 
 ###############################################################
 # Source FV3GFS workflow modules

diff --git a/ush/preamble.sh b/ush/preamble.sh
@@ -19,7 +19,7 @@
 #
 #######
 set +x
-if [[ -v '1' ]]; then
+if (( $# > 0 )); then
 	id="(${1})"
 else
 	id=""

diff --git a/workflow/applications.py b/workflow/applications.py
@@ -217,6 +217,9 @@ def _cycled_configs(self):
         if self.do_wafs:
             configs += ['wafs', 'wafsgrib2', 'wafsblending', 'wafsgcip', 'wafsgrib20p25', 'wafsblending0p25']
 
+        if self.do_aero:
+            configs += ['aeroanlinit', 'aeroanlrun', 'aeroanlfinal']
+
         return configs
 
     @property
@@ -348,6 +351,9 @@ def _get_cycled_task_names(self):
 
         gdas_gfs_common_tasks_before_fcst += ['sfcanl', 'analcalc']
 
+        if self.do_aero:
+            gdas_gfs_common_tasks_before_fcst += ['aeroanlinit', 'aeroanlrun', 'aeroanlfinal']
+
         gldas_tasks = ['gldas']
         wave_prep_tasks = ['waveinit', 'waveprep']
         wave_bndpnt_tasks = ['wavepostbndpnt', 'wavepostbndpntbll']

diff --git a/workflow/rocoto/workflow_tasks.py b/workflow/rocoto/workflow_tasks.py
@@ -16,6 +16,7 @@ class Tasks:
                    'earc', 'ecen', 'echgres', 'ediag', 'efcs',
                    'eobs', 'eomg', 'epos', 'esfc', 'eupd',
                    'atmensanalprep', 'atmensanalrun', 'atmensanalpost',
+                   'aeroanlinit', 'aeroanlrun', 'aeroanlfinal',
                    'fcst', 'post', 'ocnpost', 'vrfy', 'metp',
                    'postsnd', 'awips', 'gempak',
                    'wafs', 'wafsblending', 'wafsblending0p25',
@@ -244,7 +245,7 @@ def prep(self):
         data = f'&ROTDIR;/gdas.@Y@m@d/@H/atmos/gdas.t@Hz.atmf009{suffix}'
         dep_dict = {'type': 'data', 'data': data, 'offset': '-06:00:00'}
         deps.append(rocoto.add_dependency(dep_dict))
-        data = f'{dmpdir}/{self.cdump}{dump_suffix}.@Y@m@d/@H/{self.cdump}.t@Hz.updated.status.tm00.bufr_d'
+        data = f'{dmpdir}/{self.cdump}{dump_suffix}.@Y@m@d/@H/atmos/{self.cdump}.t@Hz.updated.status.tm00.bufr_d'
         dep_dict = {'type': 'data', 'data': data}
         deps.append(rocoto.add_dependency(dep_dict))
         dependencies = rocoto.create_dependency(dep_condition='and', dep=deps)
@@ -405,7 +406,7 @@ def atmanalprep(self):
         data = f'&ROTDIR;/gdas.@Y@m@d/@H/atmos/gdas.t@Hz.atmf009{suffix}'
         dep_dict = {'type': 'data', 'data': data, 'offset': '-06:00:00'}
         deps.append(rocoto.add_dependency(dep_dict))
-        data = f'{dmpdir}/{self.cdump}{dump_suffix}.@Y@m@d/@H/{self.cdump}.t@Hz.updated.status.tm00.bufr_d'
+        data = f'{dmpdir}/{self.cdump}{dump_suffix}.@Y@m@d/@H/atmos/{self.cdump}.t@Hz.updated.status.tm00.bufr_d'
         dep_dict = {'type': 'data', 'data': data}
         deps.append(rocoto.add_dependency(dep_dict))
         dependencies = rocoto.create_dependency(dep_condition='and', dep=deps)
@@ -450,6 +451,54 @@ def atmanalpost(self):
 
         return task
 
+    def aeroanlinit(self):
+
+        suffix = self._base["SUFFIX"]
+        dump_suffix = self._base["DUMP_SUFFIX"]
+        gfs_cyc = self._base["gfs_cyc"]
+        dmpdir = self._base["DMPDIR"]
+
+        deps = []
+        dep_dict = {'type': 'metatask', 'name': 'gdaspost', 'offset': '-06:00:00'}
+        deps.append(rocoto.add_dependency(dep_dict))
+        data = f'&ROTDIR;/gdas.@Y@m@d/@H/atmos/gdas.t@Hz.atmf009{suffix}'
+        dep_dict = {'type': 'data', 'data': data, 'offset': '-06:00:00'}
+        deps.append(rocoto.add_dependency(dep_dict))
+        data = f'{dmpdir}/{self.cdump}{dump_suffix}.@Y@m@d/@H/atmos/{self.cdump}.t@Hz.updated.status.tm00.bufr_d'
+        dep_dict = {'type': 'data', 'data': data}
+        deps.append(rocoto.add_dependency(dep_dict))
+        dependencies = rocoto.create_dependency(dep_condition='and', dep=deps)
+
+        resources = self.get_resource('aeroanlinit')
+        task = create_wf_task('aeroanlinit', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies)
+        return task
+
+    def aeroanlrun(self):
+
+        deps = []
+        dep_dict = {'type': 'task', 'name': f'{self.cdump}aeroanlinit'}
+        deps.append(rocoto.add_dependency(dep_dict))
+        dependencies = rocoto.create_dependency(dep=deps)
+
+        resources = self.get_resource('aeroanlrun')
+        task = create_wf_task('aeroanlrun', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies)
+
+        return task
+
+    def aeroanlfinal(self):
+
+        deps = []
+        dep_dict = {'type': 'task', 'name': f'{self.cdump}aeroanlrun'}
+        deps.append(rocoto.add_dependency(dep_dict))
+        dep_dict = {'type': 'cycleexist', 'offset': '-06:00:00'}
+        deps.append(rocoto.add_dependency(dep_dict))
+        dependencies = rocoto.create_dependency(dep_condition='and', dep=deps)
+
+        resources = self.get_resource('aeroanlfinal')
+        task = create_wf_task('aeroanlfinal', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies)
+
+        return task
+
     def gldas(self):
 
         deps = []
@@ -470,15 +519,14 @@ def fcst(self):
                     'cycled': self._fcst_cycled}
 
         try:
-            task = fcst_map[self.app_config.mode]
+            task = fcst_map[self.app_config.mode]()
         except KeyError:
             raise NotImplementedError(f'{self.app_config.mode} is not a valid type.\n' +
                                       'Currently supported forecast types are:\n' +
                                       f'{" | ".join(fcst_map.keys())}')
 
         return task
 
-    @property
     def _fcst_forecast_only(self):
         dependencies = []
 
@@ -531,7 +579,6 @@ def _fcst_forecast_only(self):
 
         return task
 
-    @property
     def _fcst_cycled(self):
 
         dep_dict = {'type': 'task', 'name': f'{self.cdump}sfcanl'}
@@ -985,7 +1032,7 @@ def atmensanalprep(self):
         data = f'&ROTDIR;/gdas.@Y@m@d/@H/atmos/gdas.t@Hz.atmf009{suffix}'
         dep_dict = {'type': 'data', 'data': data, 'offset': '-06:00:00'}
         deps.append(rocoto.add_dependency(dep_dict))
-        data = f'{dmpdir}/{self.cdump}{dump_suffix}.@Y@m@d/@H/{self.cdump}.t@Hz.updated.status.tm00.bufr_d'
+        data = f'{dmpdir}/{self.cdump}{dump_suffix}.@Y@m@d/@H/atmos/{self.cdump}.t@Hz.updated.status.tm00.bufr_d'
         dep_dict = {'type': 'data', 'data': data}
         deps.append(rocoto.add_dependency(dep_dict))
         dependencies = rocoto.create_dependency(dep_condition='and', dep=deps)

diff --git a/workflow/rocoto/workflow_xml.py b/workflow/rocoto/workflow_xml.py
@@ -113,7 +113,7 @@ def _get_cycledefs_cycled(self):
         sdate = self._base['SDATE'].strftime('%Y%m%d%H%M')
         edate = self._base['EDATE'].strftime('%Y%m%d%H%M')
         interval = self._base.get('INTERVAL', '06:00:00')
-        strings = [f'\t<cycledef group="gdas" >{sdate} {edate} {interval}</cycledef>']
+        strings = [f'\t<cycledef group="gdas" >{sdate} {edate} {interval}</cycledef>\n']
 
         if self._app_config.gfs_cyc != 0:
             sdate_gfs = self._base['SDATE_GFS'].strftime('%Y%m%d%H%M')

diff --git a/workflow/setup_expt.py b/workflow/setup_expt.py
@@ -228,7 +228,7 @@ def input_args():
     cycled.add_argument('--nens', help='number of ensemble members',
                         type=int, required=False, default=20)
     cycled.add_argument('--app', help='UFS application', type=str,
-                        choices=['ATM', 'ATMW'], required=False, default='ATM')
+                        choices=['ATM', 'ATMW', 'ATMA'], required=False, default='ATM')
 
     # forecast only mode additional arguments
     forecasts.add_argument('--app', help='UFS application', type=str, choices=[