Merge f0f64e6 into b8cbcf3

activefires_pp/fire_notifications.py

@@ -440,7 +440,7 @@ def read_geojson_data(filename):
 def get_recipients_for_region(recipients, region_code):
     """Get the recipients lists applicable to the region."""
     for region_id in recipients:
-        rcode = recipients[region_id]['knkod']
+        rcode = recipients[region_id]['Kod_omr']
         if rcode == region_code:
             recpt = RecipientDataStruct()
             recpt._set_recipients(recipients[region_id]['recipients'],

activefires_pp/geometries_from_shapefiles.py

@@ -0,0 +1,82 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright (c) 2021 Adam.Dybbroe
+
+# Author(s):
+
+#   Adam Dybbroe <Firstname.Lastname at smhi.se>
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Loading shapefile geometries from files.
+"""
+
+from glob import glob
+import os
+import cartopy.io.shapereader as shpreader
+import pycrs
+
+
+class ShapeGeometry(object):
+    """Geometry from a shape file."""
+
+    def __init__(self, shapefilepath, globstr='*.shp'):
+        self.filepaths = _get_shapefile_paths(shapefilepath, globstr)
+
+        self.geometries = None
+        self.attributes = None
+        self._get_proj()
+
+    def load(self):
+        """Load the geometries and the associated attributes."""
+        records = []
+        for filepath in self.filepaths:
+            records = records + [r for r in shpreader.Reader(filepath).records()]
+
+        self._records = records
+        self._load_member_from_records('geometries', 'geometry')
+        self._load_member_from_records('attributes', 'attributes')
+
+    def _get_proj(self):
+        """Get and return the Proj.4 string."""
+
+        self.proj4str = []
+        for filepath in self.filepaths:
+            prj_filename = filepath.strip('.shp') + '.prj'
+            crs = pycrs.load.from_file(prj_filename)
+            self.proj4str.append(crs.to_proj4())
+
+        first_proj4_str = self.proj4str[0]
+        for proj4_str in self.proj4str[1:]:
+            if proj4_str != first_proj4_str:
+                return
+
+        self.proj4str = first_proj4_str
+
+    def _load_member_from_records(self, class_member, record_type):
+        """Load a member of the shapely.geometry object and set the corresponding class member."""
+        setattr(self, class_member, [getattr(rec, record_type) for rec in self._records])
+
+
+def _get_shapefile_paths(path, globstr='*.shp'):
+    """Get full filepaths for all shapefiles in directory or simply return the paths as a list.
+
+    From a path to a directory with shapefiles or a full file path, 
+    return list of file paths for all shapefiles.
+    """
+    if os.path.isfile(path):
+        return [path]
+
+    return glob(os.path.join(path, globstr))

activefires_pp/post_processing.py

@@ -24,9 +24,8 @@
 """
 
 import socket
-import yaml
-from yaml import UnsafeLoader
-from trollsift import Parser
+from trollsift import Parser, globify
+import time
 import pandas as pd
 from datetime import datetime, timedelta
 import numpy as np
@@ -42,16 +41,13 @@
 from posttroll.publisher import NoisyPublisher
 import pyproj
 from matplotlib.path import Path
-import time
-import cartopy.io.shapereader as shpreader
 import shapely
-import pycrs
 
 from activefires_pp.utils import datetime_from_utc_to_local
 from activefires_pp.utils import get_local_timezone
 from activefires_pp.utils import json_serial
 from activefires_pp.utils import read_config
-
+from activefires_pp.geometries_from_shapefiles import ShapeGeometry
 
 # M-band output:
 # column 1: latitude of fire pixel (degrees)
@@ -80,33 +76,6 @@
 logging.getLogger("fiona").setLevel(logging.WARNING)
 
 
-class ShapeGeometry(object):
-    """Geometry from a shape file."""
-
-    def __init__(self, shapefilepath):
-        self.filepath = shapefilepath
-        self.geometries = None
-        self.attributes = None
-        self.proj4str = self._get_proj()
-
-    def load(self):
-        """Load the geometries and the associated attributes."""
-        self._records = [r for r in shpreader.Reader(self.filepath).records()]
-        self._load_member_from_records('geometries', 'geometry')
-        self._load_member_from_records('attributes', 'attributes')
-
-    def _get_proj(self):
-        """Get and return the Proj.4 string."""
-
-        prj_filename = self.filepath.strip('.shp') + '.prj'
-        crs = pycrs.load.from_file(prj_filename)
-        return crs.to_proj4()
-
-    def _load_member_from_records(self, class_member, record_type):
-        """Load a member of the shapely.geometry object and set the corresponding class member."""
-        setattr(self, class_member, [getattr(rec, record_type) for rec in self._records])
-
-
 class ActiveFiresShapefileFiltering(object):
     """Reading, filtering and writing Active Fire detections.
 
@@ -202,14 +171,14 @@ def fires_filtering(self, shapefile, start_geometries_index=1, inside=True):
         else:
             logger.debug("Number of detections after filtering on Polygon: %d", len(self._afdata))
 
-    def get_regional_filtermasks(self, shapefile):
+    def get_regional_filtermasks(self, shapefile, globstr):
         """Get the regional filter masks from the shapefile."""
         detections = self._afdata
 
         lons = detections.longitude.values
         lats = detections.latitude.values
 
-        shape_geom = ShapeGeometry(shapefile)
+        shape_geom = ShapeGeometry(shapefile, globstr)
         shape_geom.load()
 
         p__ = pyproj.Proj(shape_geom.proj4str)
@@ -332,7 +301,6 @@ def get_mask_from_multipolygon(points, geometry):
     mask = pth.contains_points(points)
 
     if sum(mask) == len(points):
-        print("All points inside test area!")
         return mask
 
     for shape in geometry.geoms[1:]:
@@ -394,6 +362,9 @@ def __init__(self, configfile, shp_boarders, shp_mask, regional_filtermask=None)
         self.outfile_pattern_regional = self.options.get('geojson_file_pattern_regional')
         self.output_dir = self.options.get('output_dir', '/tmp')
 
+        frmt = self.options['regional_shapefiles_format']
+        self.regional_shapefiles_globstr = globify(frmt)
+
         self.listener = None
         self.publisher = None
         self.loop = False
@@ -488,7 +459,8 @@ def run(self):
                 # FIXME! If afdata is empty (len=0) then it seems all data are inside all regions!
                 af_shapeff = ActiveFiresShapefileFiltering(afdata=afdata, platform_name=platform_name,
                                                            timezone=self.timezone)
-                regional_fmask = af_shapeff.get_regional_filtermasks(self.regional_filtermask)
+                regional_fmask = af_shapeff.get_regional_filtermasks(self.regional_filtermask,
+                                                                     globstr=self.regional_shapefiles_globstr)
                 regional_messages = self.regional_fires_filtering_and_publishing(msg, regional_fmask, af_shapeff)
                 for region_msg in regional_messages:
                     logger.debug("Sending message: %s", str(region_msg))
@@ -512,7 +484,7 @@ def regional_fires_filtering_and_publishing(self, msg, regional_fmask, afsff_obj
                 continue
 
             regions_with_detections = regions_with_detections + 1
-            fmda['region_name'] = regional_fmask[region_name]['attributes']['KNKOD']
+            fmda['region_name'] = regional_fmask[region_name]['attributes']['Kod_omr']
 
             out_filepath = os.path.join(self.output_dir, pout.compose(fmda))
             logger.debug("Output file path = %s", out_filepath)
@@ -626,7 +598,7 @@ def get_filename_from_uri(uri):
 def generate_posttroll_topic(output_topic, region=None):
     """Create the topic for the posttroll message to publish."""
     if region:
-        output_topic = output_topic + '/Regional/' + region['attributes']['KNKOD']
+        output_topic = output_topic + '/Regional/' + region['attributes']['Kod_omr']
     else:
         output_topic = output_topic + '/National'
 
@@ -646,6 +618,6 @@ def prepare_posttroll_message(input_msg, region=None):
     # FIXME! Check that the region_name is stored as a unicode string!
     if region:
         to_send['region_name'] = region['attributes']['Testomr']
-        to_send['region_code'] = region['attributes']['KNKOD']
+        to_send['region_code'] = region['attributes']['Kod_omr']
 
     return to_send

activefires_pp/tests/test_fire_notifications.py

@@ -80,15 +80,15 @@
 recipients:
   Area1-name:
     subject: "My subject"
-    knkod: '0999'
+    Kod_omr: '0999'
     name: 'Name of my area 1'
     recipients:
       - active-fires-sms-0999@mydomain.xx
     recipients_attachment: [active-fires-0999@mydomain.xx, ]
 
   Area2-name:
     subject: "My subject"
-    knkod: '0114'
+    Kod_omr: '0114'
     name: 'Name of my area 2'
     recipients: [active-fires-sms-0998@mydomain.xx, ]
     recipients_attachment: [active-fires-0998@mydomain.xx, ]
@@ -194,10 +194,10 @@ def test_get_options_regional_filtering(self, setup_comm, read_config, gethostna
                     'smtp_server': 'smtp.mydomain.xx', 'domain': 'mydomain.xx',
                     'sender': 'active-fires@mydomain.xx',
                     'recipients': {
-                        'Area1-name': {'subject': 'My subject', 'knkod': '0999', 'name': 'Name of my area 1',
+                        'Area1-name': {'subject': 'My subject', 'Kod_omr': '0999', 'name': 'Name of my area 1',
                                        'recipients': ['active-fires-sms-0999@mydomain.xx'],
                                        'recipients_attachment': ['active-fires-0999@mydomain.xx']},
-                        'Area2-name': {'subject': 'My subject', 'knkod': '0114', 'name':
+                        'Area2-name': {'subject': 'My subject', 'Kod_omr': '0114', 'name':
                                        'Name of my area 2', 'recipients': ['active-fires-sms-0998@mydomain.xx'],
                                        'recipients_attachment': ['active-fires-0998@mydomain.xx']}},
                     'max_number_of_fires_in_sms': 3, 'fire_data': ['power', 'observation_time'],

activefires_pp/tests/test_fires_filtering.py

@@ -30,10 +30,7 @@
 import numpy as np
 import io
 from datetime import datetime
-import pycrs
-import cartopy.io.shapereader
 
-from activefires_pp.post_processing import ShapeGeometry
 from activefires_pp.post_processing import ActiveFiresShapefileFiltering
 from activefires_pp.post_processing import ActiveFiresPostprocessing
 from activefires_pp.post_processing import COL_NAMES
@@ -84,6 +81,8 @@
                   'AFIMG_{platform:s}_d{start_time:%Y%m%d_t%H%M%S%f}_e{end_hour:%H%M%S%f}_b{orbit:s}_c{processing_time:%Y%m%d%H%M%S%f}_cspp_dev.txt',
                   'geojson_file_pattern_national': 'AFIMG_{platform:s}_d{start_time:%Y%m%d_t%H%M%S}.geojson',
                   'geojson_file_pattern_regional': 'AFIMG_{platform:s}_d{start_time:%Y%m%d_t%H%M%S}_{region_name:s}.geojson',
+                  'regional_shapefiles_format': 'omr_{region_code:s}_Buffer.{ext:s}',
+
                   'output_dir': '/path/where/the/filtered/results/will/be/stored',
                   'timezone': 'Europe/Stockholm'}
 
@@ -93,14 +92,11 @@
 MY_FILE_PATTERN = ("AFIMG_{platform:s}_d{start_time:%Y%m%d_t%H%M%S%f}_e{end_hour:%H%M%S%f}_" +
                    "b{orbit:s}_c{processing_time:%Y%m%d%H%M%S%f}_cspp_dev.txt")
 
-TEST_CRS_PROJ = ('+proj=utm +ellps=GRS80 +a=6378137.0 +rf=298.257222101 +pm=0 +x_0=500000.0 ' +
-                 '+y_0=0.0 +lon_0=15.0 +lat_0=0.0 +units=m +axis=enu +no_defs')
-
 TEST_REGIONAL_MASK = {}
 TEST_REGIONAL_MASK['Bergslagen (RRB)'] = {'mask': np.array([False, False, False, False, False, False, False, False, False,
                                                             False, False, False, False,  True, False, False, False, False]),
                                           'attributes': {'Join_Count': 2, 'TARGET_FID': 142,
-                                                         'KNKOD': '1438', 'KNNAMN': 'Dals-Ed',
+                                                         'Kod_omr': '1438', 'KNNAMN': 'Dals-Ed',
                                                          'LANDAREAKM': 728.0, 'KNBEF96': 5287.0,
                                                          'OBJECTID': 1080804, 'Datum_Tid': '2016-06-13',
                                                          'Testomr': 'Bergslagen (RRB)',
@@ -109,7 +105,7 @@
 TEST_REGIONAL_MASK['Västerviks kommun'] = {'mask': np.array([False, False, False, False, False, False, False, False, False,
                                                              False, False, False, False, False, False, False, False, False]),
                                            'attributes': {'Join_Count': 30, 'TARGET_FID': 85,
-                                                          'KNKOD': '0883', 'KNNAMN': 'Västervik',
+                                                          'Kod_omr': '0883', 'KNNAMN': 'Västervik',
                                                           'LANDAREAKM': 1870.5, 'KNBEF96': 39579.0,
                                                           'OBJECTID': 1079223, 'Datum_Tid': '2016-06-13',
                                                           'Testomr': 'Västerviks kommun',
@@ -121,56 +117,6 @@
 FAKE_MASK2 = np.array([True, False,  True])
 
 
-class MyMockCrs(object):
-    def __init__(self):
-        self._crs_proj = TEST_CRS_PROJ
-
-    def to_proj4(self):
-        return self._crs_proj
-
-
-class fake_geometry_records(object):
-    def __init__(self):
-        self.geometry = 'Some geom'
-        self.attributes = {'attr1': 1, 'attr2': 'myname'}
-
-
-def fake_get_records():
-    """Fake retrieving the generator class of the Geometry records in a shapefile."""
-    num = 0
-    while num < 10:
-        yield fake_geometry_records()
-        num = num + 1
-
-
-@patch('pycrs.load.from_file')
-def test_shape_geometry_init(load_from_file):
-    """Test creating the ShapeGeometry object."""
-
-    load_from_file.return_value = MyMockCrs()
-    shpgeom = ShapeGeometry('/my/shape/file/path/myshapefile.sph')
-
-    assert shpgeom.proj4str == TEST_CRS_PROJ
-
-
-@patch('pycrs.load.from_file')
-def test_shape_geometry_loading(load_from_file):
-    """Test loading the geometries and attributes from the shapefile."""
-
-    load_from_file.return_value = MyMockCrs()
-    shpgeom = ShapeGeometry('/my/shape/file/path/myshapefile.sph')
-
-    with patch('cartopy.io.shapereader.Reader') as MockShpReader:
-        MockShpReader.return_value.records.return_value = fake_get_records()
-        shpgeom.load()
-
-    assert shpgeom.geometries is not None
-    assert shpgeom.attributes is not None
-
-    assert len(shpgeom.geometries) == 10
-    assert len(shpgeom.attributes) == 10
-
-
 @patch('activefires_pp.post_processing._read_data')
 def test_add_start_and_end_time_to_active_fires_data_utc(readdata):
     """Test adding start and end times to the active fires data."""

activefires_pp/tests/test_messaging.py

@@ -45,6 +45,7 @@
                   'AFIMG_{platform:s}_d{start_time:%Y%m%d_t%H%M%S%f}_e{end_hour:%H%M%S%f}_b{orbit:s}_c{processing_time:%Y%m%d%H%M%S%f}_cspp_dev.txt',
                   'geojson_file_pattern_national': 'AFIMG_{platform:s}_d{start_time:%Y%m%d_t%H%M%S}.geojson',
                   'geojson_file_pattern_regional': 'AFIMG_{platform:s}_d{start_time:%Y%m%d_t%H%M%S}_{region_name:s}.geojson',
+                  'regional_shapefiles_format': 'omr_{region_code:s}_Buffer.{ext:s}',
                   'output_dir': '/path/where/the/filtered/results/will/be/stored'}
 
 
@@ -78,7 +79,7 @@ def test_prepare_posttroll_message(setup_comm, get_config, gethostname):
 
     input_msg = Message.decode(rawstr=TEST_MSG)
 
-    fake_region_mask = {'attributes': {'KNKOD': '9999',
+    fake_region_mask = {'attributes': {'Kod_omr': '9999',
                                        'Testomr': 'Some area description'}}
     res_msg = afpp._generate_output_message(test_filepath, input_msg, region=fake_region_mask)