Fix and simplify online archiving NOAA-EMC#2673

DavidHuber-NOAA · Jun 13, 2024 · 345b5c9 · 345b5c9
1 parent 603a4a8
commit 345b5c9
Show file tree

Hide file tree

Showing 2 changed files with 130 additions and 111 deletions.
diff --git a/parm/archive/arcdir.yaml.j2 b/parm/archive/arcdir.yaml.j2
@@ -4,116 +4,143 @@
 {% set head = RUN + ".t" + cycle_HH + "z." %}
 
 # Select data to store in the ARCDIR and VFYARC from deterministic runs
-Base: &base # GDAS, GFS, ENKFGDAS, or ENKFGFS
-    common:
-        mkdir:
-            - "{{ ARCDIR }}"
-
-# Common files to be added to both the gfs and gdas keys below
-Deterministic: &deterministic
-    cyclone:
-        copy:
-            # Cyclone forecasts, produced for both gdas and gfs cycles
-            ## Only created if tracking is on and there were systems to track
-            {% if path_exists(COM_ATMOS_TRACK ~ "/atcfunix." ~ RUN ~ "." ~ cycle_YMDH) %}
-            - ["{{ COM_ATMOS_TRACK }}/atcfunix.{{ RUN }}.{{ cycle_YMDH }}", "{{ ARCDIR }}/atcfunix.{{ RUN }}.{{ cycle_YMDH }}"]
-            - ["{{ COM_ATMOS_TRACK }}/atcfunixp.{{ RUN }}.{{ cycle_YMDH }}", "{{ ARCDIR }}/atcfunixp.{{ RUN }}.{{ cycle_YMDH }}"]
-            {% endif %}
-
-            # Cyclone tracking data
-            {% for basin in ["epac", "natl"] %}
-                {% if path_exists(COM_ATMOS_TRACK + "/" + basin) %}
-            - ["{{ COM_ATMOS_TRACK }}/{{ basin }}", "{{ ARCDIR }}/{{ basin }}"]
-                {% endif %}
-            {% endfor %}
-
-    {% if MODE == "cycled" %}
-    analysis:
-        copy:
-            # Analysis data (if we are running in cycled mode)
-            - ["{{ COM_ATMOS_GRIB_1p00 }}/{{ head }}pgrb2.1p00.anl", "{{ ARCDIR }}/pgbanl.{{ RUN }}.{{ cycle_YMDH }}.grib2"]
-
-            {% if DO_JEDIATMVAR %}
-            - ["{{ COM_ATMOS_ANALYSIS }}/{{ head }}atmstat", "{{ ARCDIR }}/atmstat.{{ RUN }}.{{ cycle_YMDH }}"]
-            {% else %}
-            - ["{{ COM_ATMOS_ANALYSIS }}/{{ head }}gsistat", "{{ ARCDIR }}/gsistat.{{ RUN }}.{{ cycle_YMDH }}"]
-            {% endif %}
+mkdir:
+    - "{{ ARCDIR }}"
+    {% if DO_FIT2OBS %}
+        {% set VFYARC = ROTDIR + "/vrfyarch" %}
+    - "{{ VFYARC }}/{{ RUN }}.{{ cycle_YMD }}/{{ cycle_HH }}"
+    {% endif %}
 
-            {% if DO_JEDISNOWDA %}
-            - ["{{ COM_SNOW_ANALYSIS }}/{{ head }}snowstat.tgz", "{{ ARCDIR }}/snowstat.{{ RUN }}.{{ cycle_YMDH }}.tgz"]
-            {% endif %}
+# This file set will contain all source-destination pairs to send to the FileHandler for copying
+{% set file_set = [] %}
+
+# Deterministic files
+{% if "enkf" not in RUN %}
+    # Common files to be added to both the gfs and gdas keys below
+    {% set det_files = [] %}
+    # Cyclone forecasts, produced for both gdas and gfs cycles
+    ## Only created if tracking is on and there were systems to track
+    {% if path_exists(COM_ATMOS_TRACK ~ "/atcfunix." ~ RUN ~ "." ~ cycle_YMDH) %}
+        {% do det_files.append([COM_ATMOS_TRACK ~ "/atcfunix." ~ RUN ~ "." ~ cycle_YMDH,
+                                ARCDIR ~"/atcfunix." ~ RUN ~ "." ~ cycle_YMDH ]) %}
+        {% do det_files.append([COM_ATMOS_TRACK ~ "/atcfunixp." ~ RUN ~ "." ~ cycle_YMDH,
+                                ARCDIR ~ "/atcfunixp." ~ RUN ~ "." ~ cycle_YMDH]) %}
+    {% endif %}
 
-            {% if AERO_ANL_CDUMP == RUN or AERO_ANL_CDUMP == "both" %}
-            - ["{{ COM_CHEM_ANALYSIS }}/{{ head }}aerostat", "{{ ARCDIR }}/aerostat.{{ RUN }}.{{ cycle_YMDH }}"]
-            {% endif %}
+    # Cyclone tracking data
+    {% for basin in ["epac", "natl"] %}
+        {% if path_exists(COM_ATMOS_TRACK + "/" + basin) %}
+            {% do det_files.append([COM_ATMOS_TRACK ~ "/" ~ basin,
+                                    ARCDIR ~ "/" ~ basin ]) %}
+        {% endif %}
+    {% endfor %}
+
+    # Deterministic analysis files (generated for cycled experiments)
+    {% set det_anl_files = [] %}
+    # Analysis data (if we are running in cycled mode)
+        {% do det_anl_files.append([COM_ATMOS_GRIB_1p00 ~ "/" ~ head ~ "pgrb2.1p00.anl",
+                                    ARCDIR ~ "/pgbanl." ~ RUN ~ "." ~ cycle_YMDH ~ ".grib2"]) %}
+
+    {% if DO_JEDIATMVAR %}
+        {% do det_anl_files.append([COM_ATMOS_ANALYSIS ~ "/" ~ head ~ "atmstat",
+                                    ARCDIR ~ "/atmstat." ~ RUN ~ "." ~ cycle_YMDH ]) %}
+    {% else %}
+        {% do det_anl_files.append([COM_ATMOS_ANALYSIS ~ "/" ~ head ~ "gsistat",
+                                    ARCDIR ~ "/gsistat." ~ RUN ~ "." ~ cycle_YMDH ]) %}
+    {% endif %}
 
+    {% if DO_JEDISNOWDA %}
+        {% do det_anl_files.append([COM_SNOW_ANALYSIS ~ "/" ~ head ~ "snowstat.tgz",
+                                    ARCDIR ~ "/snowstat." ~ RUN ~ "." ~ cycle_YMDH ~ ".tgz"]) %}
     {% endif %}
 
-{% if RUN == "gfs" %}
-gfs:  # GFS specific
-    <<: *base
-    <<: *deterministic
+    {% if AERO_ANL_CDUMP == RUN or AERO_ANL_CDUMP == "both" %}
+        {% do det_anl_files.append([COM_CHEM_ANALYSIS ~ "/" ~ head ~ "aerostat",
+                                    ARCDIR ~ "/aerostat." ~ RUN ~ "." ~ cycle_YMDH ]) %}
+    {% endif %}
 
-    gfs:
-        copy:
-            {% for fhr in range(0, FHMAX_GFS + 1, FHOUT_GFS) %}
-            - ["{{ COM_ATMOS_GRIB_1p00 }}/{{ head }}pgrb2.1p00.f{{ '%03d' % fhr }}", "{{ ARCDIR }}/pgbf{{ '%02d' % fhr }}.{{ RUN }}.{{ cycle_YMDH }}.grib2"]
-            {% endfor %}
+    # GFS-specific files
+    {% set gfs_files = [] %}
+    {% for fhr in range(0, FHMAX_GFS + 1, FHOUT_GFS) %}
+        {% do gfs_files.append([COM_ATMOS_GRIB_1p00 ~ "/" ~ head ~ "pgrb2.1p00.f" ~ '%03d'|format(fhr),
+                                ARCDIR ~ "/pgbf" ~ '%02d'|format(fhr) ~ "." ~ RUN ~ "." ~ cycle_YMDH ~ ".grib2"]) %}
+    {% endfor %}
+
+    # Cyclone genesis data (only present if there are storms)
+    {% if path_exists(COM_ATMOS_GENESIS ~ "/storms.gfso.atcf_gen." ~ cycle_YMDH) %}
+        {% do gfs_files.append([COM_ATMOS_GENESIS ~ "/storms.gfso.atcf_gen." ~ cycle_YMDH,
+                                ARCDIR ~ "/storms.gfso.atcf_gen." ~ cycle_YMDH ]) %}
+        {% do gfs_files.append([COM_ATMOS_GENESIS ~ "/storms.gfso.atcf_gen.altg." ~ cycle_YMDH,
+                                ARCDIR ~ "/storms.gfso.atcf_gen.altg." ~ cycle_YMDH ]) %}
+    {% endif %}
 
-            # Cyclone genesis data (only present if there are storms)
-            {% if path_exists(COM_ATMOS_GENESIS ~ "/storms.gfso.atcf_gen." ~ cycle_YMDH) %}
-            - ["{{ COM_ATMOS_GENESIS }}/storms.gfso.atcf_gen.{{ cycle_YMDH }}", "{{ ARCDIR }}/storms.gfso.atcf_gen.{{ cycle_YMDH }}"]
-            - ["{{ COM_ATMOS_GENESIS }}/storms.gfso.atcf_gen.altg.{{ cycle_YMDH }}", "{{ ARCDIR }}/storms.gfso.atcf_gen.altg.{{ cycle_YMDH }}"]
-            {% endif %}
+    {% if path_exists(COM_ATMOS_GENESIS ~ "/trak.gfso.atcfunix." ~ cycle_YMDH) %}
+        {% do gfs_files.append([COM_ATMOS_GENESIS ~ "/trak.gfso.atcfunix." ~ cycle_YMDH,
+                                ARCDIR ~ "/trak.gfso.atcfunix." ~ cycle_YMDH ]) %}
+        {% do gfs_files.append([COM_ATMOS_GENESIS ~ "/trak.gfso.atcfunix.altg." ~ cycle_YMDH,
+                                ARCDIR ~ "/trak.gfso.atcfunix.altg." ~ cycle_YMDH ]) %}
+    {% endif %}
 
-            {% if path_exists(COM_ATMOS_GENESIS ~ "/trak.gfso.atcfunix." ~ cycle_YMDH) %}
-            - ["{{ COM_ATMOS_GENESIS }}/trak.gfso.atcfunix.{{ cycle_YMDH }}", "{{ ARCDIR }}/trak.gfso.atcfunix.{{ cycle_YMDH }}"]
-            - ["{{ COM_ATMOS_GENESIS }}/trak.gfso.atcfunix.altg.{{ cycle_YMDH }}", "{{ ARCDIR }}/trak.gfso.atcfunix.altg.{{ cycle_YMDH }}"]
-            {% endif %}
+    # GFS Fit2Obs data
+    {% set fit2obs_files = [] %}
+    {% for fhr in range(0, FHMAX_FITS + 1, 6) %}
+        {% set sfcfile = "/" + head + "sfcf" + '%03d'|format(fhr) + ".nc" %}
+        {% set sigfile = "/" + head + "atmf" + '%03d'|format(fhr) + ".nc" %}
+        {% do fit2obs_files.append([COM_ATMOS_HISTORY ~ "/" ~ sfcfile,
+                                    VFYARC ~ "/" ~ RUN ~ "." ~ cycle_YMD ~ "/" ~ cycle_HH ~ "/" ~ sfcfile ]) %}
+        {% do fit2obs_files.append([COM_ATMOS_HISTORY ~ "/" ~ sigfile,
+                                    VFYARC ~ "/" ~ RUN ~ "." ~ cycle_YMD ~ "/" ~ cycle_HH ~ "/" ~ sigfile ]) %}
+    {% endfor %}
+
+    # GDAS-specific files
+    {% set gdas_files = [] %}
+    {% for fhr in range(0, FHMAX + 1, FHOUT) %}
+        {% do gdas_files.append([COM_ATMOS_GRIB_1p00 ~ "/" ~ head ~ "pgrb2.1p00.f" ~ '%03d'|format(fhr),
+                                 ARCDIR ~ "/pgbf" ~ '%02d'|format(fhr) ~ "." ~ RUN ~ "." ~ cycle_YMDH ~ ".grib2"]) %}
+    {% endfor %}
+
+    # Now append the necessary file pairs to file_set
+    # Common deterministic files
+    {% set file_set = file_set + det_files %}
+    {% if MODE == "cycled" %}
+        {% set file_set = file_set + det_anl_files %}
+    {% endif %}
 
-    {% if DO_FIT2OBS %}
-    fit2obs:
+    # Run-specific deterministic files
+    {% if RUN == "gfs" %}
+        {% set file_set = file_set + gfs_files %}
+        # Fit2Obs files
+        {% if DO_FIT2OBS == True %}
+            {% set file_set = file_set + fit2obs_files %}
+        {% endif %}
+    {% elif RUN == "gdas" %}
+        {% set file_set = file_set + gdas_files %}
+    {% endif %}
 
-        mkdir:
-            {% set VFYARC = ROTDIR + "/vrfyarch" %}
-            - "{{ VFYARC }}/{{ RUN }}.{{ cycle_YMD }}/{{ cycle_HH }}"
+{% else %}  # End of deterministic files
+
+    # Ensemble analysis files
+    {% set enkf_files = [] %}
+    {% if DO_JEDIATMENS %}
+        {% do enkf_files.append([COM_ATMOS_ANALYSIS_ENSSTAT ~ "/" ~ head ~ "atmensstat",
+                                 ARCDIR ~ "/atmensstat." ~ RUN ~ "." ~ cycle_YMDH ]) %}
+        {% do enkf_files.append([COM_ATMOS_ANALYSIS_ENSSTAT ~ "/" ~ head ~ "atminc.ensmean.nc",
+                                 ARCDIR ~ "/atmensstat." ~ RUN ~ "." ~ cycle_YMDH ~ ".ensmean.nc"]) %}
+    {% else %}
+        {% do enkf_files.append([COM_ATMOS_ANALYSIS_ENSSTAT ~ "/" ~ head ~ "enkfstat",
+                                 ARCDIR ~ "/enkfstat." ~ RUN ~ "." ~ cycle_YMDH ]) %}
+        {% do enkf_files.append([COM_ATMOS_ANALYSIS_ENSSTAT ~ "/" ~ head ~ "gsistat.ensmean",
+                                 ARCDIR ~ "/gsistat." ~ RUN ~ "." ~ cycle_YMDH ~ ".ensmean"]) %}
+    {% endif %}
 
-        copy:
-            {% for fhr in range(0, FHMAX_FITS + 1, 6) %}
-                {% set sfcfile = "/" + head + "sfcf" + '%03d'|format(fhr) + ".nc" %}
-                {% set sigfile = "/" + head + "atmf" + '%03d'|format(fhr) + ".nc" %}
-            - ["{{COM_ATMOS_HISTORY}}/{{ sfcfile }}", "{{ VFYARC }}/{{ RUN }}.{{ cycle_YMD }}/{{ cycle_HH }}/{{ sfcfile }}"]
-            - ["{{COM_ATMOS_HISTORY}}/{{ sigfile }}", "{{ VFYARC }}/{{ RUN }}.{{ cycle_YMD }}/{{ cycle_HH }}/{{ sigfile }}"]
-            {% endfor %}
+    # Construct the final file set
+    {% set file_set = file_set + enkf_files %}
 
-    {% endif %}
 {% endif %}
 
-{% if RUN == "gdas" %}
-gdas:  # GDAS specific
-    <<: *base
-    <<: *deterministic
-    gdas:
-        copy:
-            {% for fhr in range(0, FHMAX + 1, FHOUT) %}
-            - ["{{ COM_ATMOS_GRIB_1p00 }}/{{ head }}pgrb2.1p00.f{{ '%03d' % fhr }}", "{{ ARCDIR }}/pgbf{{ '%02d' % fhr }}.{{ RUN }}.{{ cycle_YMDH }}.grib2"]
-            {% endfor %}
-{% endif %}
 
-Ensemble: &ensemble # ENKFGDAS or ENKFGFS
-    analysis:
-        copy:
-            # Copy ensemble analyses
-            {% if DO_JEDIATMENS %}
-            - ["{{ COM_ATMOS_ANALYSIS_ENSSTAT }}/{{ head }}atmensstat", "{{ ARCDIR }}/atmensstat.{{ RUN }}.{{ cycle_YMDH }}"]
-            - ["{{ COM_ATMOS_ANALYSIS_ENSSTAT }}/{{ head }}atminc.ensmean.nc", "{{ ARCDIR }}/atmensstat.{{ RUN }}.{{ cycle_YMDH }}.ensmean.nc"]
-            {% else %}
-            - ["{{ COM_ATMOS_ANALYSIS_ENSSTAT }}/{{ head }}enkfstat", "{{ ARCDIR }}/enkfstat.{{ RUN }}.{{ cycle_YMDH }}"]
-            - ["{{ COM_ATMOS_ANALYSIS_ENSSTAT }}/{{ head }}gsistat.ensmean", "{{ ARCDIR }}/gsistat.{{ RUN }}.{{ cycle_YMDH }}.ensmean"]
-            {% endif %}
-
-{% if "enkf" in RUN %}
-{{ RUN }}:
-    <<: *base
-    <<: *ensemble
-{% endif %}
+# Actually write the yaml
+copy:
+   {% for source_dest_pair in file_set %}
+   - {{ source_dest_pair }}
+   {% endfor %}
diff --git a/ush/python/pygfs/task/archive.py b/ush/python/pygfs/task/archive.py
@@ -348,19 +348,11 @@ def _construct_arcdir_set(arcdir_j2yaml, arch_dict) -> Dict:
             files need to be copied to the ARCDIR and the Fit2Obs directory.
         """
 
-        # Parse the input jinja yaml template
-        arcdir_yaml = parse_j2yaml(arcdir_j2yaml,
-                                   arch_dict,
-                                   allow_missing=True)
-
-        # Collect the needed FileHandler dicts and construct arcdir_set
-        arcdir_set = {}
-        for key, handler in arcdir_yaml[arch_dict.RUN].items():
-            # Different RUNs can have different filesets that need to be copied.
-            # Each fileset is stored as a dictionary.  Collect the contents of
-            # each (which should be 'mkdir' and/or 'copy') to produce singular
-            # mkdir and copy lists.
-            arcdir_set.update(handler)
+        # Get the FileHandler dictionary for creating directories and copying
+        # to the ARCDIR and VFYARC directories.
+        arcdir_set = parse_j2yaml(arcdir_j2yaml,
+                                  arch_dict,
+                                  allow_missing=True)
 
         return arcdir_set