Check that the National boarders shapefile actually exists before pro…

…ceeding

Signed-off-by: Adam.Dybbroe <a000680@c21856.ad.smhi.se>

activefires_pp/geometries_from_shapefiles.py

@@ -20,8 +20,7 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-"""Loading shapefile geometries from files.
-"""
+"""Loading shapefile geometries from files."""
 
 from glob import glob
 import os
@@ -33,8 +32,8 @@ class ShapeGeometry(object):
     """Geometry from a shape file."""
 
     def __init__(self, shapefilepath, globstr='*.shp'):
+        """Initialize the ShapeGeometry class."""
         self.filepaths = _get_shapefile_paths(shapefilepath, globstr)
-
         self.geometries = None
         self.attributes = None
         self._get_proj()
@@ -51,7 +50,6 @@ def load(self):
 
     def _get_proj(self):
         """Get and return the Proj.4 string."""
-
         self.proj4str = []
         for filepath in self.filepaths:
             prj_filename = filepath.strip('.shp') + '.prj'
@@ -84,4 +82,7 @@ def _get_shapefile_paths(path, globstr='*.shp'):
     if os.path.isfile(path):
         return [path]
 
-    return glob(os.path.join(path, globstr))
+    shapefile_paths = glob(os.path.join(path, globstr))
+    if len(shapefile_paths) == 0:
+        raise OSError('No matching shapefiles found on disk. Path = %s, glob-string = %s', path, globstr)
+    return shapefile_paths

activefires_pp/post_processing.py

@@ -315,9 +315,6 @@ def get_global_mask_from_shapefile(shapefile, lonlats, start_geom_index=0):
     lons, lats = lonlats
 
     logger.debug("Getting the global mask from file: shapefile file path = %s" % str(shapefile))
-    if not os.path.exists(shapefile):
-        raise OSError("Shape file does not exist! Filename = %s", shapefile)
-
     shape_geom = ShapeGeometry(shapefile)
     shape_geom.load()
 
@@ -347,7 +344,6 @@ def __init__(self, configfile, shp_boarders, shp_mask, regional_filtermask=None)
         super().__init__()
         self.shp_boarders = shp_boarders
         self.shp_filtermask = shp_mask
-        # Should check here of the shapefile actually exists on disk: FIXME!
 
         self.regional_filtermask = regional_filtermask
         self.configfile = configfile
@@ -381,6 +377,8 @@ def _setup_and_start_communication(self):
         now = datetime_utc2local(datetime.now(), self.timezone)
         logger.debug("Output times for timezone: {zone} Now = {time}".format(zone=str(self.timezone), time=now))
 
+        self._check_boarders_shapes_exists()
+
         self.listener = ListenerContainer(topics=[self.input_topic])
         self.publisher = NoisyPublisher("active_fires_postprocessing")
         self.publisher.start()
@@ -578,6 +576,11 @@ def _generate_no_fires_messages(self, input_msg, msg_string):
 
         return publish_messages
 
+    def _check_boarders_shapes_exists(self):
+        """Check that the national boarders shapefile exists on disk."""
+        if not os.path.exists(self.shp_boarders):
+            raise OSError("Shape file does not exist! Filename = %s" % self.shp_boarders)
+
     def close(self):
         """Shutdown the Active Fires postprocessing."""
         logger.info('Terminating Active Fires post processing.')

activefires_pp/tests/conftest.py

@@ -171,3 +171,13 @@ def fake_past_detections_dir(tmp_path):
         fpt.write(PAST_ALARMS_MONSTERAS2)
 
     yield file_path.parent
+
+
+@pytest.fixture
+def fake_national_boarders_shapefile(tmp_path):
+    """Write fake national boarders shape file."""
+    file_path = tmp_path / 'some_national_boarders_shape.yaml'
+    with open(file_path, 'w') as fpt:
+        fpt.write('')
+
+    yield file_path

activefires_pp/tests/test_fires_filtering.py

@@ -20,9 +20,9 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-"""
-"""
+"""Test the Fires filtering functionality."""
 
+import pytest
 from unittest.mock import patch
 import pandas as pd
 import numpy as np
@@ -75,11 +75,12 @@
 CONFIG_EXAMPLE = {'publish_topic': '/VIIRS/L2/Fires/PP',
                   'subscribe_topics': 'VIIRS/L2/AFI',
                   'af_pattern_ibands':
-                  'AFIMG_{platform:s}_d{start_time:%Y%m%d_t%H%M%S%f}_e{end_hour:%H%M%S%f}_b{orbit:s}_c{processing_time:%Y%m%d%H%M%S%f}_cspp_dev.txt',
+                  'AFIMG_{platform:s}_d{start_time:%Y%m%d_t%H%M%S%f}_e{end_hour:%H%M%S%f}' +
+                  '_b{orbit:s}_c{processing_time:%Y%m%d%H%M%S%f}_cspp_dev.txt',
                   'geojson_file_pattern_national': 'AFIMG_{platform:s}_d{start_time:%Y%m%d_t%H%M%S}.geojson',
-                  'geojson_file_pattern_regional': 'AFIMG_{platform:s}_d{start_time:%Y%m%d_t%H%M%S}_{region_name:s}.geojson',
+                  'geojson_file_pattern_regional': 'AFIMG_{platform:s}_d{start_time:%Y%m%d_t%H%M%S}_' +
+                  '{region_name:s}.geojson',
                   'regional_shapefiles_format': 'omr_{region_code:s}_Buffer.{ext:s}',
-
                   'output_dir': '/path/where/the/filtered/results/will/be/stored',
                   'timezone': 'Europe/Stockholm'}
 
@@ -90,23 +91,28 @@
                    "b{orbit:s}_c{processing_time:%Y%m%d%H%M%S%f}_cspp_dev.txt")
 
 TEST_REGIONAL_MASK = {}
-TEST_REGIONAL_MASK['Bergslagen (RRB)'] = {'mask': np.array([False, False, False, False, False, False, False, False, False,
-                                                            False, False, False, False,  True, False, False, False, False]),
+TEST_REGIONAL_MASK['Bergslagen (RRB)'] = {'mask': np.array([False, False, False, False, False,
+                                                            False, False, False, False,
+                                                            False, False, False, False,  True,
+                                                            False, False, False, False]),
                                           'attributes': {'Join_Count': 2, 'TARGET_FID': 142,
                                                          'Kod_omr': '1438', 'KNNAMN': 'Dals-Ed',
                                                          'LANDAREAKM': 728.0, 'KNBEF96': 5287.0,
                                                          'OBJECTID': 1080804, 'Datum_Tid': '2016-06-13',
                                                          'Testomr': 'Bergslagen (RRB)',
                                                          'Shape_Leng': 2131994.36042, 'Shape_Area': 53512139344.2},
                                           'all_inside_test_area': False, 'some_inside_test_area': True}
-TEST_REGIONAL_MASK['Västerviks kommun'] = {'mask': np.array([False, False, False, False, False, False, False, False, False,
-                                                             False, False, False, False, False, False, False, False, False]),
+TEST_REGIONAL_MASK['Västerviks kommun'] = {'mask': np.array([False, False, False, False, False,
+                                                             False, False, False, False,
+                                                             False, False, False, False, False,
+                                                             False, False, False, False]),
                                            'attributes': {'Join_Count': 30, 'TARGET_FID': 85,
                                                           'Kod_omr': '0883', 'KNNAMN': 'Västervik',
                                                           'LANDAREAKM': 1870.5, 'KNBEF96': 39579.0,
                                                           'OBJECTID': 1079223, 'Datum_Tid': '2016-06-13',
                                                           'Testomr': 'Västerviks kommun',
-                                                          'Shape_Leng': 251653.298274, 'Shape_Area': 2040770168.02},
+                                                          'Shape_Leng': 251653.298274,
+                                                          'Shape_Area': 2040770168.02},
                                            'all_inside_test_area': False, 'some_inside_test_area': False}
 
 FAKE_MASK1 = np.array([False, False, False, False, False, False, False, False,  True,
@@ -117,7 +123,6 @@
 @patch('activefires_pp.post_processing._read_data')
 def test_add_start_and_end_time_to_active_fires_data_utc(readdata):
     """Test adding start and end times to the active fires data."""
-
     myfilepath = TEST_ACTIVE_FIRES_FILEPATH
 
     fstream = io.StringIO(TEST_ACTIVE_FIRES_FILE_DATA)
@@ -140,7 +145,6 @@ def test_add_start_and_end_time_to_active_fires_data_utc(readdata):
 @patch('activefires_pp.post_processing._read_data')
 def test_add_start_and_end_time_to_active_fires_data_localtime(readdata):
     """Test adding start and end times to the active fires data."""
-
     myfilepath = TEST_ACTIVE_FIRES_FILEPATH
 
     fstream = io.StringIO(TEST_ACTIVE_FIRES_FILE_DATA)
@@ -243,7 +247,6 @@ def test_regional_fires_filtering(setup_comm, get_config, gethostname):
 @patch('activefires_pp.post_processing.get_global_mask_from_shapefile', side_effect=[FAKE_MASK1, FAKE_MASK2])
 def test_general_national_fires_filtering(get_global_mask, setup_comm, get_config, gethostname):
     """Test the general/basic national fires filtering."""
-
     get_config.return_value = CONFIG_EXAMPLE
     gethostname.return_value = "my.host.name"
 
@@ -278,10 +281,50 @@ def test_general_national_fires_filtering(get_global_mask, setup_comm, get_confi
 
     store_geojson.assert_called_once()
     get_output_msg.assert_called_once()
-    get_global_mask.call_count == 2
+    assert get_global_mask.call_count == 2
 
     assert isinstance(result, pd.core.frame.DataFrame)
     assert len(result) == 1
     np.testing.assert_almost_equal(result.iloc[0]['latitude'], 59.52483368)
     np.testing.assert_almost_equal(result.iloc[0]['longitude'], 17.1681633)
     assert outmsg == ["my fake output message"]
+
+
+@pytest.mark.usefixtures("fake_national_boarders_shapefile")
+@patch('socket.gethostname')
+@patch('activefires_pp.post_processing.read_config')
+@patch('activefires_pp.post_processing.ActiveFiresPostprocessing._setup_and_start_communication')
+def test_checking_national_boarders_shapefile_file_exists(setup_comm, get_config,
+                                                          gethostname, fake_national_boarders_shapefile):
+    """Test the checking of the national boarders shapefile - boarders shapefile exists."""
+    get_config.return_value = CONFIG_EXAMPLE
+    gethostname.return_value = "my.host.name"
+
+    myconfigfile = "/my/config/file/path"
+    mymask_file = "/my/shape/file/with/polygons/to/filter/out"
+
+    afpp = ActiveFiresPostprocessing(myconfigfile, fake_national_boarders_shapefile, mymask_file)
+    afpp._check_boarders_shapes_exists()
+
+    assert afpp.shp_boarders.name == 'some_national_boarders_shape.yaml'
+    assert afpp.shp_boarders.is_file()
+
+
+@patch('socket.gethostname')
+@patch('activefires_pp.post_processing.read_config')
+@patch('activefires_pp.post_processing.ActiveFiresPostprocessing._setup_and_start_communication')
+def test_checking_national_boarders_shapefile_file_nonexisting(setup_comm, get_config, gethostname):
+    """Test the checking of the national boarders shapefile - boarders shapefile does not exist."""
+    get_config.return_value = CONFIG_EXAMPLE
+    gethostname.return_value = "my.host.name"
+
+    myconfigfile = "/my/config/file/path"
+    myboarders_file = "/my/shape/file/with/country/boarders"
+    mymask_file = "/my/shape/file/with/polygons/to/filter/out"
+
+    afpp = ActiveFiresPostprocessing(myconfigfile, myboarders_file, mymask_file)
+    with pytest.raises(OSError) as exec_info:
+        afpp._check_boarders_shapes_exists()
+
+    expected = "Shape file does not exist! Filename = /my/shape/file/with/country/boarders"
+    assert str(exec_info.value) == expected