Merge 23007f4 into 63679fc

activefires_pp/api_posting.py

@@ -0,0 +1,29 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright (c) 2022 Adam.Dybbroe
+
+# Author(s):
+
+#   Adam.Dybbroe <a000680@c21856.ad.smhi.se>
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Post geojson formatted Alarms to a ReST-API
+"""
+
+
+def post_alarm(geojson_data, url):
+    """Post an Alarm to a rest-api stored as a geojson file."""
+    pass

activefires_pp/config.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright (c) 2022 Adam Dybbroe
+
+# Author(s):
+
+#   Adam Dybbroe <Firstname.Lastname at smhi.se>
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Handling the yaml configurations.
+"""
+
+import yaml
+from yaml import UnsafeLoader
+
+
+def read_config(config_filepath):
+    """Read and extract config information."""
+    with open(config_filepath, 'r') as fp_:
+        config = yaml.load(fp_, Loader=UnsafeLoader)
+
+    return config

activefires_pp/fire_notifications.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-# Copyright (c) 2021 Adam Dybbroe
+# Copyright (c) 2021, 2022 Adam Dybbroe
 
 # Author(s):
 
@@ -28,7 +28,6 @@
 from datetime import datetime
 import os
 from six.moves.urllib.parse import urlparse
-import geojson
 
 import logging
 import signal
@@ -44,7 +43,9 @@
 from email.mime.text import MIMEText
 from email import encoders
 
-from activefires_pp.utils import read_config
+from activefires_pp.config import read_config
+from activefires_pp.utils import get_filename_from_posttroll_message
+from activefires_pp.geojson_utils import read_geojson_data
 
 
 HOME = os.environ.get('HOME')
@@ -196,10 +197,7 @@ def notify_end_users(self, msg):
         """Send notifications to configured end users (mail and text messages)."""
         LOG.debug("Start sending notifications to configured end users.")
 
-        url = urlparse(msg.data.get('uri'))
-        LOG.info('File path: %s', str(url.path))
-        filename = url.path
-
+        filename = get_filename_from_posttroll_message(msg)
         ffdata = read_geojson_data(filename)
         if not ffdata:
             return None
@@ -381,10 +379,7 @@ def notify_end_users(self, msg):
         """Send notifications to configured end users (mail and text messages)."""
         LOG.debug("Start sending notifications to configured end users.")
 
-        url = urlparse(msg.data.get('uri'))
-        LOG.info('File path: %s', str(url.path))
-        filename = url.path
-
+        filename = get_filename_from_posttroll_message(msg)
         ffdata = read_geojson_data(filename)
         if not ffdata:
             return None
@@ -427,16 +422,6 @@ def _create_output_message(msg, topic, recipients):
     return Message(topic, 'info', to_send)
 
 
-def read_geojson_data(filename):
-    """Read Geo json data from file."""
-    if filename.endswith('.geojson') and os.path.exists(filename):
-        # Read the file:
-        with open(filename, "r") as fpt:
-            return geojson.load(fpt)
-    else:
-        LOG.warning("No filename to read: %s", filename)
-
-
 def get_recipients_for_region(recipients, region_code):
     """Get the recipients lists applicable to the region."""
     for region_id in recipients:

activefires_pp/geojson_utils.py

@@ -0,0 +1,90 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright (c) 2022 Adam Dybbroe
+
+# Author(s):
+
+#   Adam Dybbroe <Firstname.Lastname@smhi.se>
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Geojson utilities.
+"""
+
+import os
+import geojson
+import json
+import logging
+from trollsift import Parser, globify
+import pytz
+from datetime import datetime, timedelta
+import numpy as np
+
+LOG = logging.getLogger(__name__)
+
+
+def read_geojson_data(filename):
+    """Read Geo json data from file."""
+    if str(filename).endswith('.geojson') and filename.exists():
+        # Read the file:
+        try:
+            with open(filename, "r") as fpt:
+                return geojson.load(fpt)
+        except json.decoder.JSONDecodeError:
+            LOG.exception("Geojson file invalid and cannot be read: %s", str(filename))
+    else:
+        LOG.error("No valid filename to read: %s", str(filename))
+
+
+def get_recent_geojson_files(path, pattern, time_interval):
+    """Get all geojson files with filtered active fire detections (=triggered alarms) since *dtime*."""
+    dtime_start = time_interval[0]
+    dtime_end = time_interval[1]
+    if not dtime_end:
+        dtime_end = datetime.utcnow()
+
+    p__ = Parser(pattern)
+    files = path.glob(globify(pattern))
+    dtimes = []
+    fnames = []
+    for gjson_file in files:
+        fname = gjson_file.name
+        res = p__.parse(fname)
+        if res['start_time'] > dtime_start and res['start_time'] < dtime_end:
+            #print("File: %s" % fname)
+            dtimes.append(res['start_time'])
+            fnames.append(fname)
+
+    dtimes = np.array(dtimes)
+    fnames = np.array(fnames)
+
+    idx = dtimes.argsort()
+    files = np.take(fnames, idx)
+    return files.tolist()
+
+
+def store_geojson_alarm(fires_alarms_dir, file_parser, idx, alarm):
+    """Store the fire alarm to a geojson file."""
+    utc = pytz.timezone('utc')
+    start_time = datetime.fromisoformat(alarm["features"]["properties"]["observation_time"])
+    platform_name = alarm["features"]["properties"]["platform_name"]
+    start_time = start_time.astimezone(utc).replace(tzinfo=None)
+    fname = file_parser.compose({'start_time': start_time, 'id': idx,
+                                 'platform_name': platform_name})
+    output_filename = fires_alarms_dir / fname
+    with open(output_filename, 'w') as fpt:
+        geojson.dump(alarm, fpt)
+
+    return output_filename

activefires_pp/post_processing.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-# Copyright (c) 2021 - 2022 Adam.Dybbroe
+# Copyright (c) 2021 - 2022 Adam.Dybbro
 
 # Author(s):
 
@@ -46,7 +46,7 @@
 from activefires_pp.utils import datetime_utc2local
 from activefires_pp.utils import get_local_timezone_offset
 from activefires_pp.utils import json_serial
-from activefires_pp.utils import read_config
+from activefires_pp.config import read_config
 from activefires_pp.geometries_from_shapefiles import ShapeGeometry
 
 # M-band output: