Merge cb9670d into 455af3e

adybbroe · Sep 5, 2022 · c10413c · c10413c
2 parents 455af3e + cb9670d
commit c10413c
Show file tree

Hide file tree

Showing 11 changed files with 499 additions and 139 deletions.
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
@@ -2,6 +2,6 @@
 
  - [ ] Closes #xxxx <!-- remove if there is no corresponding issue, which should only be the case for minor changes -->
  - [ ] Tests added <!-- for all bug fixes or enhancements -->
- - [ ] Tests passed: Passes ``pytest pyspectral`` <!-- for all non-documentation changes) -->
+ - [ ] Tests passed: Passes ``pytest`` <!-- for all non-documentation changes) -->
  - [ ] Passes ``flake8`` <!-- remove if you did not edit any Python files -->
  - [ ] Fully documented <!-- remove if this change should not be visible to users, e.g., if it is an internal clean-up, or if this is part of a larger project that will be documented later -->
diff --git a/activefires_pp/api_posting.py b/activefires_pp/api_posting.py
@@ -1,11 +1,11 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-# Copyright (c) 2022 Adam.Dybbroe
+# Copyright (c) 2022 Adam Dybbroe
 
 # Author(s):
 
-#   Adam.Dybbroe <a000680@c21856.ad.smhi.se>
+#   Adam Dybbroe <Firstname.Lastname@smhi.se>
 
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -20,10 +20,32 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-"""Post geojson formatted Alarms to a ReST-API
-"""
+"""Post geojson formatted Alarms to a ReST-API."""
 
+import logging
+import requests
 
-def post_alarm(geojson_data, url):
+# Data payload to be posted - Example:
+# {"type": "Feature", "geometry": {"type": "Point", "coordinates": [15.860621, 61.403141]},
+# "properties": {"power": 3.09576535, "tb": 328.81933594, "confidence": 8,
+#                "observation_time": "2022-08-02T03:27:43.850000",
+#                "platform_name": "NOAA-20",  "related_detection": false}}
+
+LOG = logging.getLogger(__name__)
+
+
+def post_alarm(geojson_data, api_url, xauth=None):
     """Post an Alarm to a rest-api stored as a geojson file."""
-    pass
+    if xauth is None:
+        headers = {"Content-Type": "application/json; charset=utf-8"}
+    else:
+        headers = {"Content-Type": "application/json; charset=utf-8",
+                   "x-auth-satellite-alarm": xauth}
+
+    response = requests.post(api_url,
+                             headers=headers,
+                             json=geojson_data)
+
+    LOG.info("Alarm posted: Response = %s", str(response))
+    LOG.debug("Status code = %d", response.status_code)
+    response.raise_for_status()
diff --git a/activefires_pp/config.py b/activefires_pp/config.py
@@ -20,11 +20,10 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-"""Handling the yaml configurations.
-"""
+"""Handling the yaml configurations."""
 
 import yaml
-from yaml import UnsafeLoader
+from yaml import UnsafeLoader, Loader
 
 
 def read_config(config_filepath):
@@ -33,3 +32,11 @@ def read_config(config_filepath):
         config = yaml.load(fp_, Loader=UnsafeLoader)
 
     return config
+
+
+def get_xauthentication_token(xauth_filepath):
+    """Get the X-Authentication-token needed for posting to the API."""
+    with open(xauth_filepath, 'r') as fp_:
+        tokens = yaml.load(fp_, Loader=Loader)
+
+    return tokens['xauth_tokens']['x-auth-satellite-alarm']
diff --git a/activefires_pp/spatiotemporal_alarm_filtering.py b/activefires_pp/spatiotemporal_alarm_filtering.py
@@ -36,10 +36,12 @@
 
 """
 
+import os
 import logging
 import signal
 from queue import Empty
 from threading import Thread
+from requests.exceptions import HTTPError
 from posttroll.listener import ListenerContainer
 from posttroll.message import Message
 from posttroll.publisher import NoisyPublisher
@@ -54,6 +56,8 @@
 
 from activefires_pp.utils import get_filename_from_posttroll_message
 from activefires_pp.config import read_config
+from activefires_pp.config import get_xauthentication_token
+
 from activefires_pp.geojson_utils import read_geojson_data
 from activefires_pp.geojson_utils import get_recent_geojson_files
 from activefires_pp.geojson_utils import store_geojson_alarm
@@ -84,6 +88,9 @@ def __init__(self, configfile):
 
         self.sos_alarms_file_pattern = self.options['geojson_file_pattern_alarms']
         self.restapi_url = self.options['restapi_url']
+        _xauth_filepath = get_xauthentication_filepath_from_environment()
+        self._xauth_token = get_xauthentication_token(_xauth_filepath)
+
         self.fire_alarms_dir = Path(self.options['fire_alarms_dir'])
 
         self.listener = None
@@ -102,7 +109,6 @@ def _setup_and_start_communication(self):
 
     def _set_options_from_config(self, config):
         """From the configuration on disk set the option dictionary with all metadata for real-time processing."""
-
         for item in config:
             self.options[item] = config[item]
 
@@ -180,7 +186,13 @@ def send_alarms(self, geojson_alarms, msg):
             # 1) Create the filename
             # 2) Wite to a file
             output_filename = store_geojson_alarm(self.fire_alarms_dir, p__, idx, alarm)
-            post_alarm(alarm, self.restapi_url)
+            try:
+                post_alarm(alarm['features'], self.restapi_url, self._xauth_token)
+                LOG.info('Alarm sent - status OK')
+            except HTTPError:
+                LOG.exception('Failed sending alarm!')
+                LOG.error('Data: %s', str(alarm['features']))
+
             output_message = _create_output_message(msg, self.output_topic, alarm, output_filename)
             LOG.debug("Sending message: %s", str(output_message))
             self.publisher.send(str(output_message))
@@ -200,6 +212,15 @@ def close(self):
                 LOG.exception("Couldn't stop publisher.")
 
 
+def get_xauthentication_filepath_from_environment():
+    """Get the filename with the X-Authentication-token from environment."""
+    xauth_filepath = os.environ.get('FIREALARMS_XAUTH_FILEPATH')
+    if xauth_filepath is None:
+        raise OSError("Environment variable FIREALARMS_XAUTH_FILEPATH not set!")
+
+    return xauth_filepath
+
+
 def dump_collection(idx, features):
     """Dump the list of features as a Geojson Feature Collection."""
     tmpdir = Path(DIR_SPATIAL_FILTER)
@@ -219,7 +240,7 @@ def create_alarms_from_fire_detections(fire_data, past_detections_dir, sos_alarm
     # detections in smaller parts, and create potential alarms:
 
     alarms_list = []
-    for idx, key in enumerate(gathered_fires):
+    for key in gathered_fires:
         LOG.debug("Key: %s" % key)
         fire_alarms = get_single_point_fires_as_collections(gathered_fires[key], long_fires_threshold)
 
@@ -255,7 +276,6 @@ def find_neighbours(feature, other_features, thr_dist=0.8):
 
 def gather_neighbours_to_new_collection(start_id, features, feature_collections, thr_dist=None):
     """Go through all features and gather into groups of neighbouring detections."""
-
     first_point = features[start_id]
     features.pop(start_id)
     neighbour_ids = find_neighbours(first_point, features)
@@ -306,7 +326,6 @@ def join_fire_detections(gdata):
 
 def split_large_fire_clusters(features, km_threshold):
     """Take a list of fire detection features and split in smaller clusters/chains."""
-
     num_features = len(features)
     LOG.debug("Split large fire clusters - Number of features: %d" % num_features)
     features = dict(zip(range(num_features), features))

diff --git a/activefires_pp/tests/conftest.py b/activefires_pp/tests/conftest.py
@@ -0,0 +1,173 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright (c) 2022 Adam.Dybbroe
+
+# Author(s):
+
+#   Adam.Dybbroe <a000680@c21856.ad.smhi.se>
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Fixtures for unittests."""
+
+import pytest
+
+
+TEST_YAML_CONFIG_CONTENT = """# Publish/subscribe
+subscribe_topics: /VIIRS/L2/Fires/PP/National
+publish_topic: /VIIRS/L2/Fires/PP/SOSAlarm
+
+geojson_file_pattern_alarms: sos_{start_time:%Y%m%d_%H%M%S}_{id:d}.geojson
+
+fire_alarms_dir: /path/where/the/filtered/alarms/will/be/stored
+
+restapi_url: "https://xxx.smhi.se:xxxx"
+"""
+
+TEST_YAML_TOKENS = """xauth_tokens:
+  x-auth-satellite-alarm : 'my-token'
+"""
+
+# AFIMG_NOAA-20_20210619_005803_sweden.geojson
+TEST_GEOJSON_FILE_CONTENT_MONSTERAS = """{"type": "FeatureCollection", "features":
+[{"type": "Feature", "geometry": {"type": "Point", "coordinates": [16.240452, 57.17329]},
+"properties": {"power": 4.19946575, "tb": 336.38024902, "confidence": 8,
+"observation_time": "2021-06-19T02:58:45.700000+02:00", "platform_name": "NOAA-20"}},
+{"type": "Feature", "geometry": {"type": "Point", "coordinates": [16.247334, 57.172443]},
+"properties": {"power": 5.85325146, "tb": 339.84768677, "confidence": 8,
+"observation_time": "2021-06-19T02:58:45.700000+02:00", "platform_name": "NOAA-20"}},
+{"type": "Feature", "geometry": {"type": "Point", "coordinates": [16.242519, 57.17498]},
+"properties": {"power": 3.34151864, "tb": 316.57772827, "confidence": 8,
+"observation_time": "2021-06-19T02:58:45.700000+02:00", "platform_name": "NOAA-20"}},
+{"type": "Feature", "geometry": {"type": "Point", "coordinates": [16.249384, 57.174122]},
+"properties": {"power": 3.34151864, "tb": 310.37808228, "confidence": 8,
+"observation_time": "2021-06-19T02:58:45.700000+02:00", "platform_name": "NOAA-20"}},
+{"type": "Feature", "geometry": {"type": "Point", "coordinates": [16.241102, 57.171574]},
+"properties": {"power": 3.34151864, "tb": 339.86465454, "confidence": 8,
+"observation_time": "2021-06-19T02:58:45.700000+02:00", "platform_name": "NOAA-20"}},
+{"type": "Feature", "geometry": {"type": "Point", "coordinates": [16.247967, 57.170712]},
+"properties": {"power": 3.34151864, "tb": 335.95074463, "confidence": 8,
+"observation_time": "2021-06-19T02:58:45.700000+02:00", "platform_name": "NOAA-20"}},
+{"type": "Feature", "geometry": {"type": "Point", "coordinates": [16.246538, 57.167309]},
+"properties": {"power": 3.10640526, "tb": 337.62503052, "confidence": 8,
+"observation_time": "2021-06-19T02:58:45.700000+02:00", "platform_name": "NOAA-20"}},
+{"type": "Feature", "geometry": {"type": "Point", "coordinates": [16.239674, 57.168167]},
+"properties": {"power": 3.10640526, "tb": 305.36495972, "confidence": 8,
+"observation_time": "2021-06-19T02:58:45.700000+02:00", "platform_name": "NOAA-20"}},
+{"type": "Feature", "geometry": {"type": "Point", "coordinates": [16.245104, 57.163902]},
+"properties": {"power": 3.10640526, "tb": 336.21279907, "confidence": 8,
+"observation_time": "2021-06-19T02:58:45.700000+02:00", "platform_name": "NOAA-20"}},
+{"type": "Feature", "geometry": {"type": "Point", "coordinates": [16.251965, 57.16304]},
+"properties": {"power": 2.40693879, "tb": 306.66555786, "confidence": 8,
+"observation_time": "2021-06-19T02:58:45.700000+02:00", "platform_name": "NOAA-20"}},
+{"type": "Feature", "geometry": {"type": "Point", "coordinates": [16.250517, 57.159637]},
+"properties": {"power": 2.23312426, "tb": 325.92211914, "confidence": 8,
+"observation_time": "2021-06-19T02:58:45.700000+02:00", "platform_name": "NOAA-20"}},
+{"type": "Feature", "geometry": {"type": "Point", "coordinates": [16.24366, 57.160496]},
+"properties": {"power": 1.51176202, "tb": 317.16009521, "confidence": 8,
+"observation_time": "2021-06-19T02:58:45.700000+02:00", "platform_name": "NOAA-20"}},
+{"type": "Feature", "geometry": {"type": "Point", "coordinates": [16.242212, 57.157097]},
+"properties": {"power": 1.51176202, "tb": 303.77804565, "confidence": 8,
+"observation_time": "2021-06-19T02:58:45.700000+02:00", "platform_name": "NOAA-20"}},
+{"type": "Feature", "geometry": {"type": "Point", "coordinates": [16.249069, 57.156235]},
+"properties": {"power": 2.23312426, "tb": 310.37322998, "confidence": 8,
+"observation_time": "2021-06-19T02:58:45.700000+02:00", "platform_name": "NOAA-20"}}]}"""
+
+TEST_GEOJSON_FILE_CONTENT = """{"type": "FeatureCollection", "features":
+[{"type": "Feature", "geometry": {"type": "Point", "coordinates": [23.562864, 67.341919]},
+"properties": {"power": 1.62920368, "tb": 325.2354126, "confidence": 8,
+"observation_time": "2022-06-29T14:01:08.850000", "platform_name": "NOAA-20"}},
+{"type": "Feature", "geometry": {"type": "Point", "coordinates": [23.56245, 67.347328]},
+"properties": {"power": 3.40044808, "tb": 329.46963501, "confidence": 8,
+"observation_time": "2022-06-29T14:01:08.850000", "platform_name": "NOAA-20"}},
+{"type": "Feature", "geometry": {"type": "Point", "coordinates": [23.555086, 67.343231]},
+"properties": {"power": 6.81757641, "tb": 334.62347412, "confidence": 8,
+"observation_time": "2022-06-29T14:01:08.850000", "platform_name": "NOAA-20"}}]}"""
+
+# Past alarms:
+PAST_ALARMS_MONSTERAS1 = """{"type": "FeatureCollection", "features":
+{"type": "Feature", "geometry": {"type": "Point", "coordinates": [16.246222, 57.175987]},
+"properties": {"power": 1.83814871, "tb": 302.3949585, "confidence": 8,
+"observation_time": "2021-06-19T02:07:33.050000+02:00", "platform_name": "Suomi-NPP", "related_detection": true}}}"""
+
+PAST_ALARMS_MONSTERAS2 = """{"type": "FeatureCollection", "features":
+{"type": "Feature", "geometry": {"type": "Point", "coordinates": [16.245516, 57.1651]},
+"properties": {"power": 2.94999027, "tb": 324.5098877, "confidence": 8,
+"observation_time": "2021-06-19T02:07:33.050000+02:00", "platform_name": "Suomi-NPP", "related_detection": true}}}"""
+
+PAST_ALARMS_MONSTERAS3 = """{"features": {"geometry": {"coordinates": [16.252192, 57.15242], "type": "Point"},
+"properties": {"confidence": 8, "observation_time": "2021-06-18T14:49:01.750000+02:00",
+"platform_name": "NOAA-20", "related_detection": false, "power": 2.87395763, "tb": 330.10293579},
+"type": "Feature"}, "type": "FeatureCollection"}"""
+
+
+@pytest.fixture
+def fake_token_file(tmp_path):
+    """Write fake token file."""
+    file_path = tmp_path / '.sometokenfile.yaml'
+    with open(file_path, 'w') as fpt:
+        fpt.write(TEST_YAML_TOKENS)
+
+    yield file_path
+
+
+@pytest.fixture
+def fake_yamlconfig_file(tmp_path):
+    """Write fake yaml config file."""
+    file_path = tmp_path / 'test_alarm_filtering_config.yaml'
+    with open(file_path, 'w') as fpt:
+        fpt.write(TEST_YAML_CONFIG_CONTENT)
+
+    yield file_path
+
+
+@pytest.fixture
+def fake_geojson_file_many_detections(tmp_path):
+    """Write fake geojson file with many close detections."""
+    file_path = tmp_path / 'test_afimg_NOAA-20_20210619_005803_sweden.geojson'
+    with open(file_path, 'w') as fpt:
+        fpt.write(TEST_GEOJSON_FILE_CONTENT_MONSTERAS)
+
+    yield file_path
+
+
+@pytest.fixture
+def fake_geojson_file(tmp_path):
+    """Write fake geojson file."""
+    file_path = tmp_path / 'test_afimg_20220629_120026.geojson'
+    with open(file_path, 'w') as fpt:
+        fpt.write(TEST_GEOJSON_FILE_CONTENT)
+
+    yield file_path
+
+
+@pytest.fixture
+def fake_past_detections_dir(tmp_path):
+    """Create fake directory with past detections."""
+    past_detections_dir = tmp_path / 'past_detections'
+    past_detections_dir.mkdir()
+    file_path = past_detections_dir / 'sos_20210619_000651_0.geojson'
+    with open(file_path, 'w') as fpt:
+        fpt.write(PAST_ALARMS_MONSTERAS1)
+
+    file_path = past_detections_dir / 'sos_20210619_000651_1.geojson'
+    with open(file_path, 'w') as fpt:
+        fpt.write(PAST_ALARMS_MONSTERAS2)
+
+    file_path = past_detections_dir / 'sos_20210618_124819_0.geojson'
+    with open(file_path, 'w') as fpt:
+        fpt.write(PAST_ALARMS_MONSTERAS2)
+
+    yield file_path.parent