Merge 2112414 into 455af3e

.github/PULL_REQUEST_TEMPLATE.md

@@ -2,6 +2,6 @@
 
  - [ ] Closes #xxxx <!-- remove if there is no corresponding issue, which should only be the case for minor changes -->
  - [ ] Tests added <!-- for all bug fixes or enhancements -->
- - [ ] Tests passed: Passes ``pytest pyspectral`` <!-- for all non-documentation changes) -->
+ - [ ] Tests passed: Passes ``pytest`` <!-- for all non-documentation changes) -->
  - [ ] Passes ``flake8`` <!-- remove if you did not edit any Python files -->
  - [ ] Fully documented <!-- remove if this change should not be visible to users, e.g., if it is an internal clean-up, or if this is part of a larger project that will be documented later -->

activefires_pp/api_posting.py

@@ -1,11 +1,11 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-# Copyright (c) 2022 Adam.Dybbroe
+# Copyright (c) 2022 Adam Dybbroe
 
 # Author(s):
 
-#   Adam.Dybbroe <a000680@c21856.ad.smhi.se>
+#   Adam Dybbroe <Firstname.Lastname@smhi.se>
 
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -20,10 +20,32 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-"""Post geojson formatted Alarms to a ReST-API
-"""
+"""Post geojson formatted Alarms to a ReST-API."""
 
+import logging
+import requests
 
-def post_alarm(geojson_data, url):
+# Data payload to be posted - Example:
+# {"type": "Feature", "geometry": {"type": "Point", "coordinates": [15.860621, 61.403141]},
+# "properties": {"power": 3.09576535, "tb": 328.81933594, "confidence": 8,
+#                "observation_time": "2022-08-02T03:27:43.850000",
+#                "platform_name": "NOAA-20",  "related_detection": false}}
+
+LOG = logging.getLogger(__name__)
+
+
+def post_alarm(geojson_data, api_url, xauth=None):
     """Post an Alarm to a rest-api stored as a geojson file."""
-    pass
+    if xauth is None:
+        headers = {"Content-Type": "application/json; charset=utf-8"}
+    else:
+        headers = {"Content-Type": "application/json; charset=utf-8",
+                   "x-auth-satellite-alarm": xauth}
+
+    response = requests.post(api_url,
+                             headers=headers,
+                             json=geojson_data)
+
+    LOG.info("Alarm posted: Response = %s", str(response))
+    LOG.debug("Status code = %d", response.status_code)
+    response.raise_for_status()

activefires_pp/config.py

@@ -20,8 +20,7 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-"""Handling the yaml configurations.
-"""
+"""Handling the yaml configurations."""
 
 import yaml
 from yaml import UnsafeLoader
@@ -33,3 +32,11 @@ def read_config(config_filepath):
         config = yaml.load(fp_, Loader=UnsafeLoader)
 
     return config
+
+
+def get_xauthentication_token(xauth_filepath):
+    """Get the X-Authentication-token needed for posting to the API."""
+    with open(xauth_filepath, 'r') as fp_:
+        tokens = yaml.load(fp_, Loader=UnsafeLoader)
+
+    return tokens['xauth_tokens']['x-auth-satellite-alarm']

activefires_pp/spatiotemporal_alarm_filtering.py

@@ -36,10 +36,12 @@
 
 """
 
+import os
 import logging
 import signal
 from queue import Empty
 from threading import Thread
+from requests.exceptions import HTTPError
 from posttroll.listener import ListenerContainer
 from posttroll.message import Message
 from posttroll.publisher import NoisyPublisher
@@ -54,6 +56,8 @@
 
 from activefires_pp.utils import get_filename_from_posttroll_message
 from activefires_pp.config import read_config
+from activefires_pp.config import get_xauthentication_token
+
 from activefires_pp.geojson_utils import read_geojson_data
 from activefires_pp.geojson_utils import get_recent_geojson_files
 from activefires_pp.geojson_utils import store_geojson_alarm
@@ -84,6 +88,9 @@ def __init__(self, configfile):
 
         self.sos_alarms_file_pattern = self.options['geojson_file_pattern_alarms']
         self.restapi_url = self.options['restapi_url']
+        _xauth_filepath = get_xauthentication_filepath_from_environment()
+        self._xauth_token = get_xauthentication_token(_xauth_filepath)
+
         self.fire_alarms_dir = Path(self.options['fire_alarms_dir'])
 
         self.listener = None
@@ -102,7 +109,6 @@ def _setup_and_start_communication(self):
 
     def _set_options_from_config(self, config):
         """From the configuration on disk set the option dictionary with all metadata for real-time processing."""
-
         for item in config:
             self.options[item] = config[item]
 
@@ -180,7 +186,13 @@ def send_alarms(self, geojson_alarms, msg):
             # 1) Create the filename
             # 2) Wite to a file
             output_filename = store_geojson_alarm(self.fire_alarms_dir, p__, idx, alarm)
-            post_alarm(alarm, self.restapi_url)
+            try:
+                post_alarm(alarm['features'], self.restapi_url, self._xauth_token)
+                LOG.info('Alarm sent - status OK')
+            except HTTPError:
+                LOG.exception('Failed sending alarm!')
+                LOG.error('Data: %s', str(alarm['features']))
+
             output_message = _create_output_message(msg, self.output_topic, alarm, output_filename)
             LOG.debug("Sending message: %s", str(output_message))
             self.publisher.send(str(output_message))
@@ -200,6 +212,15 @@ def close(self):
                 LOG.exception("Couldn't stop publisher.")
 
 
+def get_xauthentication_filepath_from_environment():
+    """Get the filename with the X-Authentication-token from environment."""
+    xauth_filepath = os.environ.get('FIREALARMS_XAUTH_FILEPATH')
+    if xauth_filepath is None:
+        raise OSError("Environment variable FIREALARMS_XAUTH_FILEPATH not set!")
+
+    return xauth_filepath
+
+
 def dump_collection(idx, features):
     """Dump the list of features as a Geojson Feature Collection."""
     tmpdir = Path(DIR_SPATIAL_FILTER)
@@ -219,7 +240,7 @@ def create_alarms_from_fire_detections(fire_data, past_detections_dir, sos_alarm
     # detections in smaller parts, and create potential alarms:
 
     alarms_list = []
-    for idx, key in enumerate(gathered_fires):
+    for key in gathered_fires:
         LOG.debug("Key: %s" % key)
         fire_alarms = get_single_point_fires_as_collections(gathered_fires[key], long_fires_threshold)
 
@@ -255,7 +276,6 @@ def find_neighbours(feature, other_features, thr_dist=0.8):
 
 def gather_neighbours_to_new_collection(start_id, features, feature_collections, thr_dist=None):
     """Go through all features and gather into groups of neighbouring detections."""
-
     first_point = features[start_id]
     features.pop(start_id)
     neighbour_ids = find_neighbours(first_point, features)
@@ -306,7 +326,6 @@ def join_fire_detections(gdata):
 
 def split_large_fire_clusters(features, km_threshold):
     """Take a list of fire detection features and split in smaller clusters/chains."""
-
     num_features = len(features)
     LOG.debug("Split large fire clusters - Number of features: %d" % num_features)
     features = dict(zip(range(num_features), features))

activefires_pp/tests/test_config.py

@@ -20,11 +20,12 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-"""Test getting the yaml configurations from file.
-"""
+"""Test getting the yaml configurations from file."""
 
 import pytest
 from activefires_pp.config import read_config
+from activefires_pp.config import get_xauthentication_token
+
 
 TEST_YAML_CONFIG_CONTENT = """# Publish/subscribe
 subscribe_topics: /VIIRS/L2/Fires/PP/National
@@ -37,6 +38,20 @@
 restapi_url: "https://xxx.smhi.se:xxxx"
 """
 
+TEST_YAML_TOKENS = """xauth_tokens:
+  x-auth-satellite-alarm : 'my-token'
+"""
+
+
+@pytest.fixture
+def fake_token_file(tmp_path):
+    """Write fake token file."""
+    file_path = tmp_path / '.sometokenfile.yaml'
+    with open(file_path, 'w') as fpt:
+        fpt.write(TEST_YAML_TOKENS)
+
+    yield file_path
+
 
 @pytest.fixture
 def fake_yamlconfig_file(tmp_path):
@@ -56,3 +71,9 @@ def test_get_yaml_configuration(fake_yamlconfig_file):
     assert config['geojson_file_pattern_alarms'] == 'sos_{start_time:%Y%m%d_%H%M%S}_{id:d}.geojson'
     assert config['fire_alarms_dir'] == '/path/where/the/filtered/alarms/will/be/stored'
     assert config['restapi_url'] == 'https://xxx.smhi.se:xxxx'
+
+
+def test_get_xauthentication_token(fake_token_file):
+    """Test getting the xauthentication token from a file."""
+    fake_token = get_xauthentication_token(fake_token_file)
+    assert fake_token == 'my-token'