Merge pull request #1548 from djhoese/bugfix-awips-tiled-env-prefix

Add 'environment_prefix' to AWIPS tiled writer for flexible filenames

satpy/etc/writers/awips_tiled.yaml

@@ -777,7 +777,7 @@ templates:
   glm_l2_radc:
     single_variable: false
     # OR_GLM-L2-GLMF-M6_G16_T10_e20201105150300.nc
-    filename: 'OR_GLM-L2-GLM{scene_abbr}-{scan_mode}_{platform_shortname}_T{tile_number:02d}_{end_time:%Y%m%d%H%M%S}.nc'
+    filename: '{environment_prefix}_GLM-L2-GLM{scene_abbr}-{scan_mode}_{platform_shortname}_T{tile_number:02d}_{end_time:%Y%m%d%H%M%S}.nc'
     global_attributes:
       # FIXME: This should come from the reader's metadata
       dataset_name:
@@ -951,7 +951,7 @@ templates:
   glm_l2_radf:
     single_variable: false
     # OR_GLM-L2-GLMF-M6_G16_T10_e20201105150300.nc
-    filename: 'OR_GLM-L2-GLM{scene_abbr}-{scan_mode}_{platform_shortname}_T{tile_number:02d}_{end_time:%Y%m%d%H%M%S}.nc'
+    filename: '{environment_prefix}_GLM-L2-GLM{scene_abbr}-{scan_mode}_{platform_shortname}_T{tile_number:02d}_{end_time:%Y%m%d%H%M%S}.nc'
     global_attributes:
       # FIXME: This should come from the reader's metadata
       dataset_name:

satpy/tests/writer_tests/test_awips_tiled.py

@@ -25,6 +25,18 @@
 import pytest
 
 
+def _check_production_location(ds):
+    if 'production_site' in ds.attrs:
+        prod_loc_name = 'production_site'
+    elif 'production_location' in ds.attrs:
+        prod_loc_name = 'producton_location'
+    else:
+        return
+
+    if prod_loc_name in ds.attrs:
+        assert len(ds.attrs[prod_loc_name]) == 31
+
+
 def check_required_common_attributes(ds):
     """Check common properties of the created AWIPS tiles for validity."""
     assert 'x' in ds.coords
@@ -50,6 +62,7 @@ def check_required_common_attributes(ds):
                       'number_product_tiles',
                       'product_rows', 'product_columns'):
         assert attr_name in ds.attrs
+    _check_production_location(ds)
 
     for data_arr in ds.data_vars.values():
         if data_arr.ndim == 0:
@@ -518,13 +531,23 @@ def test_basic_numbered_tiles_rgb(self):
         ['C',
          'F']
     )
-    def test_multivar_numbered_tiles_glm(self, sector):
+    @pytest.mark.parametrize(
+        "extra_kwargs",
+        [
+            {},
+            {'environment_prefix': 'AA'},
+            {'environment_prefix': 'BB', 'filename': '{environment_prefix}_{name}_GLM_T{tile_number:04d}.nc'},
+        ]
+    )
+    def test_multivar_numbered_tiles_glm(self, sector, extra_kwargs):
         """Test creating a tiles with multiple variables."""
         import xarray as xr
         from satpy.writers.awips_tiled import AWIPSTiledWriter
         from xarray import DataArray
         from pyresample.geometry import AreaDefinition
         from pyresample.utils import proj4_str_to_dict
+        import os
+        os.environ['ORGANIZATION'] = '1' * 50
         w = AWIPSTiledWriter(base_dir=self.base_dir, compress=True)
         area_def = AreaDefinition(
             'test',
@@ -570,8 +593,10 @@ def test_multivar_numbered_tiles_glm(self, sector):
         })
 
         w.save_datasets([ds1, ds2, ds3, dqf], sector_id='TEST', source_name="TESTS",
-                        tile_count=(3, 3), template='glm_l2_rad{}'.format(sector.lower()))
-        all_files = glob(os.path.join(self.base_dir, '*_GLM*.nc'))
+                        tile_count=(3, 3), template='glm_l2_rad{}'.format(sector.lower()),
+                        **extra_kwargs)
+        fn_glob = self._get_glm_glob_filename(extra_kwargs)
+        all_files = glob(os.path.join(self.base_dir, fn_glob))
         assert len(all_files) == 9
         for fn in all_files:
             ds = xr.open_dataset(fn, mask_and_scale=False)
@@ -580,3 +605,11 @@ def test_multivar_numbered_tiles_glm(self, sector):
                 assert ds.attrs['time_coverage_end'] == end_time.strftime('%Y-%m-%dT%H:%M:%S.%fZ')
             else:  # 'F'
                 assert ds.attrs['time_coverage_end'] == end_time.strftime('%Y-%m-%dT%H:%M:%SZ')
+
+    @staticmethod
+    def _get_glm_glob_filename(extra_kwargs):
+        if 'filename' in extra_kwargs:
+            return 'BB*_GLM*.nc'
+        elif 'environment_prefix' in extra_kwargs:
+            return 'AA*_GLM*.nc'
+        return 'DR*_GLM*.nc'

satpy/writers/awips_tiled.py

@@ -216,6 +216,7 @@
 import os
 import logging
 import string
+import warnings
 import sys
 from datetime import datetime, timedelta
 from collections import namedtuple
@@ -915,10 +916,20 @@ def _global_production_location(self, input_metadata):
         del input_metadata
         org = os.environ.get('ORGANIZATION', None)
         if org is not None:
-            return org
-        LOG.warning('environment ORGANIZATION not set for .production_location attribute, using hostname')
-        import socket
-        return socket.gethostname()  # FUTURE: something more correct but this will do for now
+            prod_location = org
+        else:
+            LOG.warning('environment ORGANIZATION not set for .production_location attribute, using hostname')
+            import socket
+            prod_location = socket.gethostname()  # FUTURE: something more correct but this will do for now
+
+        if len(prod_location) > 31:
+            warnings.warn("Production location attribute is longer than 31 "
+                          "characters (AWIPS limit). Set it to a smaller "
+                          "value with the 'ORGANIZATION' environment "
+                          "variable. Defaults to hostname and is currently "
+                          "set to '{}'.".format(prod_location))
+            prod_location = prod_location[:31]
+        return prod_location
 
     _global_production_site = _global_production_location
 
@@ -1397,6 +1408,7 @@ def get_filename(self, template, area_def, tile_info, sector_id, **kwargs):
                 columns=area_def.width,
                 sector_id=sector_id,
                 tile_id=tile_info.tile_id,
+                tile_number=tile_info.tile_number,
                 **kwargs)
         except RuntimeError:
             # the user didn't provide a specific filename, use the template
@@ -1445,7 +1457,7 @@ def save_datasets(self, datasets, sector_id=None,
                       lettered_grid=False, num_subtiles=None,
                       use_end_time=False, use_sector_reference=False,
                       template='polar', check_categories=True,
-                      extra_global_attrs=None,
+                      extra_global_attrs=None, environment_prefix='DR',
                       compute=True, **kwargs):
         """Write a series of DataArray objects to multiple NetCDF4 Tile files.
 
@@ -1464,6 +1476,12 @@ def save_datasets(self, datasets, sector_id=None,
             source_name (str): Name of producer of these files (ex. "SSEC").
                 This name is used to create the output filename for some
                 templates.
+            environment_prefix (str): Prefix of filenames for some templates.
+                For operational real-time data this is usually "OR", "OT" for
+                test data, "IR" for test system real-time data, and "IT" for
+                test system test data. This defaults to "DR" for "Developer
+                Real-time" to avoid anyone accidentally producing files that
+                could be mistaken for the operational system.
             tile_count (tuple): For numbered tiles only, how many tile rows
                 and tile columns to produce. Default to ``(1, 1)``, a single
                 giant tile. Either ``tile_count``, ``tile_size``, or
@@ -1535,6 +1553,7 @@ def save_datasets(self, datasets, sector_id=None,
                                                source_name)
             output_filename = self.get_filename(template, area_def,
                                                 tile_info, sector_id,
+                                                environment_prefix=environment_prefix,
                                                 **ds_info)
             self.check_tile_exists(output_filename)
             # TODO: Provide attribute caching for things that likely won't change (functools lrucache)