Skip to content

Commit

Permalink
Add specification of file name format of the regional shapefiles
Browse files Browse the repository at this point in the history 
Signed-off-by: Adam.Dybbroe <a000680@c21856.ad.smhi.se>
  • Loading branch information
Adam.Dybbroe committed Apr 28, 2021
1 parent 7324f53 commit ca742a0
Show file tree
Hide file tree
Showing 6 changed files with 26 additions and 17 deletions.
10 changes: 5 additions & 5 deletions activefires_pp/geometries_from_shapefiles.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,8 @@
class ShapeGeometry(object):
"""Geometry from a shape file."""

def __init__(self, shapefilepath):
self.filepaths = _get_shapefile_paths(shapefilepath)
def __init__(self, shapefilepath, globstr='*.shp'):
self.filepaths = _get_shapefile_paths(shapefilepath, globstr)

self.geometries = None
self.attributes = None
Expand Down Expand Up @@ -70,13 +70,13 @@ def _load_member_from_records(self, class_member, record_type):
setattr(self, class_member, [getattr(rec, record_type) for rec in self._records])


def _get_shapefile_paths(path):
def _get_shapefile_paths(path, globstr='*.shp'):
"""Get full filepaths for all shapefiles in directory or simply return the paths as a list.
From a path to a directory with shapefiles or a full file path,
return list of file paths for all shapefiles.
"""
if os.path.isfile(path):
return [path]
else:
return glob(os.path.join(path, '*shp'))

return glob(os.path.join(path, globstr))
18 changes: 11 additions & 7 deletions activefires_pp/post_processing.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
import socket
import yaml
from yaml import UnsafeLoader
from trollsift import Parser
from trollsift import Parser, globify
import pandas as pd
from datetime import datetime, timedelta
import numpy as np
Expand Down Expand Up @@ -173,14 +173,14 @@ def fires_filtering(self, shapefile, start_geometries_index=1, inside=True):
else:
logger.debug("Number of detections after filtering on Polygon: %d", len(self._afdata))

def get_regional_filtermasks(self, shapefile):
def get_regional_filtermasks(self, shapefile, globstr):
"""Get the regional filter masks from the shapefile."""
detections = self._afdata

lons = detections.longitude.values
lats = detections.latitude.values

shape_geom = ShapeGeometry(shapefile)
shape_geom = ShapeGeometry(shapefile, globstr)
shape_geom.load()

p__ = pyproj.Proj(shape_geom.proj4str)
Expand Down Expand Up @@ -365,6 +365,9 @@ def __init__(self, configfile, shp_boarders, shp_mask, regional_filtermask=None)
self.outfile_pattern_regional = self.options.get('geojson_file_pattern_regional')
self.output_dir = self.options.get('output_dir', '/tmp')

frmt = self.options['regional_shapefiles_format']
self.regional_shapefiles_globstr = globify(frmt)

self.listener = None
self.publisher = None
self.loop = False
Expand Down Expand Up @@ -459,7 +462,8 @@ def run(self):
# FIXME! If afdata is empty (len=0) then it seems all data are inside all regions!
af_shapeff = ActiveFiresShapefileFiltering(afdata=afdata, platform_name=platform_name,
timezone=self.timezone)
regional_fmask = af_shapeff.get_regional_filtermasks(self.regional_filtermask)
regional_fmask = af_shapeff.get_regional_filtermasks(self.regional_filtermask,
globstr=self.regional_shapefiles_globstr)
regional_messages = self.regional_fires_filtering_and_publishing(msg, regional_fmask, af_shapeff)
for region_msg in regional_messages:
logger.debug("Sending message: %s", str(region_msg))
Expand All @@ -483,7 +487,7 @@ def regional_fires_filtering_and_publishing(self, msg, regional_fmask, afsff_obj
continue

regions_with_detections = regions_with_detections + 1
fmda['region_name'] = regional_fmask[region_name]['attributes']['KNKOD']
fmda['region_name'] = regional_fmask[region_name]['attributes']['Kod_omr']

out_filepath = os.path.join(self.output_dir, pout.compose(fmda))
logger.debug("Output file path = %s", out_filepath)
Expand Down Expand Up @@ -597,7 +601,7 @@ def get_filename_from_uri(uri):
def generate_posttroll_topic(output_topic, region=None):
"""Create the topic for the posttroll message to publish."""
if region:
output_topic = output_topic + '/Regional/' + region['attributes']['KNKOD']
output_topic = output_topic + '/Regional/' + region['attributes']['Kod_omr']
else:
output_topic = output_topic + '/National'

Expand All @@ -617,6 +621,6 @@ def prepare_posttroll_message(input_msg, region=None):
# FIXME! Check that the region_name is stored as a unicode string!
if region:
to_send['region_name'] = region['attributes']['Testomr']
to_send['region_code'] = region['attributes']['KNKOD']
to_send['region_code'] = region['attributes']['Kod_omr']

return to_send
6 changes: 4 additions & 2 deletions activefires_pp/tests/test_fires_filtering.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,8 @@
'AFIMG_{platform:s}_d{start_time:%Y%m%d_t%H%M%S%f}_e{end_hour:%H%M%S%f}_b{orbit:s}_c{processing_time:%Y%m%d%H%M%S%f}_cspp_dev.txt',
'geojson_file_pattern_national': 'AFIMG_{platform:s}_d{start_time:%Y%m%d_t%H%M%S}.geojson',
'geojson_file_pattern_regional': 'AFIMG_{platform:s}_d{start_time:%Y%m%d_t%H%M%S}_{region_name:s}.geojson',
'regional_shapefiles_format': 'omr_{region_code:s}_Buffer.{ext:s}',

'output_dir': '/path/where/the/filtered/results/will/be/stored',
'timezone': 'Europe/Stockholm'}

Expand All @@ -94,7 +96,7 @@
TEST_REGIONAL_MASK['Bergslagen (RRB)'] = {'mask': np.array([False, False, False, False, False, False, False, False, False,
False, False, False, False, True, False, False, False, False]),
'attributes': {'Join_Count': 2, 'TARGET_FID': 142,
'KNKOD': '1438', 'KNNAMN': 'Dals-Ed',
'Kod_omr': '1438', 'KNNAMN': 'Dals-Ed',
'LANDAREAKM': 728.0, 'KNBEF96': 5287.0,
'OBJECTID': 1080804, 'Datum_Tid': '2016-06-13',
'Testomr': 'Bergslagen (RRB)',
Expand All @@ -103,7 +105,7 @@
TEST_REGIONAL_MASK['Västerviks kommun'] = {'mask': np.array([False, False, False, False, False, False, False, False, False,
False, False, False, False, False, False, False, False, False]),
'attributes': {'Join_Count': 30, 'TARGET_FID': 85,
'KNKOD': '0883', 'KNNAMN': 'Västervik',
'Kod_omr': '0883', 'KNNAMN': 'Västervik',
'LANDAREAKM': 1870.5, 'KNBEF96': 39579.0,
'OBJECTID': 1079223, 'Datum_Tid': '2016-06-13',
'Testomr': 'Västerviks kommun',
Expand Down
3 changes: 2 additions & 1 deletion activefires_pp/tests/test_messaging.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@
'AFIMG_{platform:s}_d{start_time:%Y%m%d_t%H%M%S%f}_e{end_hour:%H%M%S%f}_b{orbit:s}_c{processing_time:%Y%m%d%H%M%S%f}_cspp_dev.txt',
'geojson_file_pattern_national': 'AFIMG_{platform:s}_d{start_time:%Y%m%d_t%H%M%S}.geojson',
'geojson_file_pattern_regional': 'AFIMG_{platform:s}_d{start_time:%Y%m%d_t%H%M%S}_{region_name:s}.geojson',
'regional_shapefiles_format': 'omr_{region_code:s}_Buffer.{ext:s}',
'output_dir': '/path/where/the/filtered/results/will/be/stored'}


Expand Down Expand Up @@ -78,7 +79,7 @@ def test_prepare_posttroll_message(setup_comm, get_config, gethostname):

input_msg = Message.decode(rawstr=TEST_MSG)

fake_region_mask = {'attributes': {'KNKOD': '9999',
fake_region_mask = {'attributes': {'Kod_omr': '9999',
'Testomr': 'Some area description'}}
res_msg = afpp._generate_output_message(test_filepath, input_msg, region=fake_region_mask)

Expand Down
4 changes: 2 additions & 2 deletions examples/fire_notifier_regional.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,14 +11,14 @@ sender: active-fires@mydomain.xx
recipients:
Area1-name:
subject: "My subject"
knkod: '0999'
Kod_omr: '0999'
name: 'Name of my area 1'
recipient: active-fires-sms-0999@mydomain.xx
recipients_attachment: active-fires-0999@mydomain.xx

Area2-name:
subject: "My subject"
knkod: '0998'
Kod_omr: '0998'
name: 'Name of my area 2'
recipient: active-fires-sms-0998@mydomain.xx
recipients_attachment: active-fires-0998@mydomain.xx
Expand Down
2 changes: 2 additions & 0 deletions examples/fires_pp.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ af_pattern_ibands: AFIMG_{platform:s}_d{start_time:%Y%m%d_t%H%M%S%f}_e{end_hour:
geojson_file_pattern_national: AFIMG_{platform:s}_d{start_time:%Y%m%d_t%H%M%S}.geojson
geojson_file_pattern_regional: AFIMG_{platform:s}_d{start_time:%Y%m%d_t%H%M%S}_{region_name:s}.geojson

regional_shapefiles_format: omr_{region_code:s}_Buffer.{ext:s}

output_dir: /path/where/the/filtered/results/will/be/stored

timezone: Europe/Stockholm
Expand Down

0 comments on commit ca742a0

Please sign in to comment.