Merge branch 'main' into discard-spurious-detections

.github/workflows/ci.yaml

@@ -10,10 +10,10 @@ jobs:
       fail-fast: true
       matrix:
         os: ["ubuntu-latest", "macos-latest"]
-        python-version: ["3.9", "3.10", "3.11"]
+        python-version: ["3.10", "3.11", "3.12"]
         experimental: [false]
         include:
-          - python-version: "3.11"
+          - python-version: "3.12"
             os: "ubuntu-latest"
             experimental: true
 

activefires_pp/spatiotemporal_alarm_filtering.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-# Copyright (c) 2022, 2023 Adam Dybbroe
+# Copyright (c) 2022, 2023, 2024 Adam Dybbroe
 
 # Author(s):
 
@@ -51,7 +51,7 @@
 from datetime import datetime, timedelta
 
 from geopy import distance
-from geojson import FeatureCollection, dump
+from geojson import FeatureCollection
 from itertools import combinations
 from pathlib import Path
 
@@ -166,6 +166,7 @@ def signal_shutdown(self, *args, **kwargs):
 
     def run(self):
         """Run the spatiotemporal alarm filtering."""
+        product_list = self.options.get('products')
         while self.loop:
             try:
                 msg = self.listener.output_queue.get(timeout=1)
@@ -180,6 +181,9 @@ def run(self):
                 elif msg.type not in ['file', 'collection', 'dataset']:
                     LOG.debug("Message type not supported: %s", str(msg.type))
                     continue
+                elif product_list and msg.data.get('product', 'unknown') not in product_list:
+                    LOG.debug("Product %s not supported/requested. Ignore.", str(msg.data.get('product')))
+                    continue
 
                 generated_alarms = self.spatio_temporal_alarm_filtering(msg)
                 if generated_alarms:
@@ -256,16 +260,6 @@ def get_xauthentication_filepath_from_environment():
     return xauth_filepath
 
 
-def dump_collection(idx, features):
-    """Dump the list of features as a Geojson Feature Collection."""
-    tmpdir = Path(DIR_SPATIAL_FILTER)
-    fname = 'sos_alarm_{index}.geojson'.format(index=idx)
-    output_filename = tmpdir / fname
-    feature_collection = FeatureCollection(features)
-    with open(output_filename, 'w') as fpt:
-        dump(feature_collection, fpt)
-
-
 def create_alarms_from_fire_detections(fire_data, past_detections_dir, sos_alarms_file_pattern,
                                        time_space_thresholds):
     """Create alarm(s) from a set of detections."""
@@ -487,11 +481,9 @@ def check_if_fire_should_trigger_alarm(gjson_data, past_alarms_dir, sos_alarms_f
 
 def distance_and_time_from_geojson_position(position, filepath):
     """Read the geojson data and get the observation time and the distance to a position given as input."""
-    lon0, lat0 = position
     gjdata = read_geojson_data(filepath)
-    lon, lat = gjdata["geometry"]["coordinates"]
     # Get distance to this fire point:
-    dist = distance.distance((lat0, lon0), (lat, lon)).kilometers
+    dist = get_distance_between_two_points(position, gjdata["geometry"]["coordinates"])
 
     obstime = datetime.fromisoformat(gjdata["properties"]["observation_time"])
     utc = pytz.timezone('utc')
@@ -500,6 +492,13 @@ def distance_and_time_from_geojson_position(position, filepath):
     return obstime, dist
 
 
+def get_distance_between_two_points(geo_point1, geo_point2):
+    """Get the distance in km on the earth between two (lon,lat) points."""
+    lon0, lat0 = geo_point1
+    lon, lat = geo_point2
+    return distance.distance((lat0, lon0), (lat, lon)).kilometers
+
+
 def _create_output_message(msg, topic, geojson, filename):
     """Create the output message from the input message and the geojson payload."""
     to_send = msg.data.copy()

activefires_pp/tests/conftest.py

@@ -29,6 +29,9 @@
 subscribe_topics: /VIIRS/L2/Fires/PP/National
 publish_topic: /VIIRS/L2/Fires/PP/SOSAlarm
 
+products:
+  - afimg
+
 geojson_file_pattern_alarms: sos_{start_time:%Y%m%d_%H%M%S}_{id:d}.geojson
 
 fire_alarms_dir: /path/where/the/filtered/alarms/will/be/stored

activefires_pp/tests/test_config.py

@@ -29,6 +29,7 @@
 def test_get_yaml_configuration_for_alarm_filtering(fake_yamlconfig_file):
     """Test read and get the yaml configuration from file for alarm filtering."""
     config = read_config(fake_yamlconfig_file)
+    assert config['products'] == ['afimg']
     assert config['subscribe_topics'] == '/VIIRS/L2/Fires/PP/National'
     assert config['publish_topic'] == '/VIIRS/L2/Fires/PP/SOSAlarm'
     assert config['geojson_file_pattern_alarms'] == 'sos_{start_time:%Y%m%d_%H%M%S}_{id:d}.geojson'

activefires_pp/tests/test_spatiotemporal_alarm_filtering.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-# Copyright (c) 2022 Adam Dybbroe
+# Copyright (c) 2022, 2023 Adam Dybbroe
 
 # Author(s):
 
@@ -30,6 +30,7 @@
 
 from activefires_pp.geojson_utils import read_geojson_data
 from activefires_pp.spatiotemporal_alarm_filtering import create_alarms_from_fire_detections
+from activefires_pp.spatiotemporal_alarm_filtering import get_distance_between_two_points
 from activefires_pp.spatiotemporal_alarm_filtering import join_fire_detections
 from activefires_pp.spatiotemporal_alarm_filtering import split_large_fire_clusters
 from activefires_pp.spatiotemporal_alarm_filtering import create_one_detection_from_collection
@@ -152,13 +153,6 @@
 "observation_time": "2021-06-18T14:49:01.750000+02:00", "platform_name": "NOAA-20"}}]}"""
 
 
-CONFIG_EXAMPLE = {'subscribe_topics': '/VIIRS/L2/Fires/PP/National',
-                  'publish_topic': '/VIIRS/L2/Fires/PP/SOSAlarm',
-                  'geojson_file_pattern_alarms': 'sos_{start_time:%Y%m%d_%H%M%S}_{id:d}.geojson',
-                  'fire_alarms_dir': '/path/where/the/filtered/alarms/will/be/stored',
-                  'restapi_url': 'https://xxx.smhi.se:xxxx'}
-
-
 def test_join_fire_detections_large_fire(fake_geojson_file_many_detections):
     """Test create alarm from set of fire detections."""
     ffdata = read_geojson_data(fake_geojson_file_many_detections)
@@ -383,6 +377,7 @@ def test_alarm_filter_runner_init(setup_comm,
     assert alarm_runner.restapi_url == 'https://xxx.smhi.se:xxxx'
     assert alarm_runner.options == {'subscribe_topics': ['/VIIRS/L2/Fires/PP/National'],
                                     'publish_topic': '/VIIRS/L2/Fires/PP/SOSAlarm',
+                                    'products': ['afimg'],
                                     'geojson_file_pattern_alarms': 'sos_{start_time:%Y%m%d_%H%M%S}_{id:d}.geojson',
                                     'fire_alarms_dir': '/path/where/the/filtered/alarms/will/be/stored',
                                     'restapi_url': 'https://xxx.smhi.se:xxxx',
@@ -514,3 +509,33 @@ def test_check_and_set_threshold_wrong_threshold(setup_comm,
 
     expected = alarm_runner._log_message_per_threshold[thr_type][1]
     assert str(exec_info.value) == expected
+
+# @pytest.mark.parametrize()
+
+
+@pytest.mark.parametrize("lonlats, expected",
+                         [((12.5, 62.5),
+                           135.02288790715303),
+                          ((13.5, 68.5),
+                           542.596822830585),
+                          ]
+                         )
+def test_get_distance_between_two_points_input_okay(lonlats, expected):
+    """Test the derivation of distance between two lon,lat points."""
+    point1 = (13.438972, 63.634071)
+    point2 = lonlats[0], lonlats[1]
+
+    dist_km = get_distance_between_two_points(point1, point2)
+    assert dist_km == pytest.approx(expected)
+
+
+def test_get_distance_between_two_points_input_invalid():
+    """Test the derivation of distance between two lon,lat points."""
+    point1 = (13.438972, 63.634071)
+    point2 = (386845.899472, 6420007.169359)
+
+    with pytest.raises(ValueError) as exec_info:
+        _ = get_distance_between_two_points(point1, point2)
+
+    expected = "Latitude must be in the [-90; 90] range."
+    assert str(exec_info.value) in expected