Fixing bug updating the detection id while running more than one day

activefires_pp/geojson_utils.py

@@ -31,8 +31,8 @@
 from trollsift import Parser, globify
 import pytz
 from datetime import datetime
-import numpy as np
 
+import numpy as np
 from activefires_pp.utils import json_serial
 
 logger = logging.getLogger(__name__)

activefires_pp/logger.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-# Copyright (c) 2021 Adam.Dybbroe
+# Copyright (c) 2021, 2023 Adam.Dybbroe
 
 # Author(s):
 
@@ -20,8 +20,7 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-"""The log handling.
-"""
+"""The log handling."""
 
 import logging
 import logging.config

activefires_pp/post_processing.py

@@ -615,11 +615,8 @@ def _initialize_fire_detection_id(self):
     def update_fire_detection_id(self):
         """Update the fire detection ID registry."""
         now = datetime.utcnow()
-        tdelta = now - self._fire_detection_id['date']
-        if tdelta.total_seconds() > 24*3600:
-            self._initialize_fire_detection_id()
-        elif tdelta.total_seconds() > 0 and self._fire_detection_id['date'].day != now.day:
-            self._initialize_fire_detection_id()
+        if self._fire_detection_id['date'].date() < now.date():
+            self._fire_detection_id = {'date': datetime.utcnow(), 'counter': 0}
 
         self._fire_detection_id['counter'] = self._fire_detection_id['counter'] + 1
 

activefires_pp/spatiotemporal_alarm_filtering.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-# Copyright (c) 2022 Adam Dybbroe
+# Copyright (c) 2022, 2023 Adam Dybbroe
 
 # Author(s):
 
@@ -121,7 +121,7 @@ def _check_and_set_thresholds_from_config(self, config):
 
         self._log_message_per_threshold = {
             'hour_threshold': ("Threshold defining when a new detection should trigger an " +
-                               "alarm on the same spot (in hours): ",
+                               "alarm on the same spot (in hours): %3.1f",
                                "Threshold in time is missing!"),
             'long_fires_threshold_km': ("Threshold defining the maximum extention of a fire before " +
                                         "dividing it in smaller pieces (in km): %3.1f",

activefires_pp/tests/conftest.py

@@ -63,20 +63,8 @@
 
 output_dir: /path/where/the/filtered/results/will/be/stored
 
-filepath_detection_id_cache: /path/to/the/detection_id/cache/fire_detection_id_cache.txt
-
 timezone: Europe/Stockholm
 
-geojson-national:
-  - kelvin:
-      file_pattern: AFIMG_{platform:s}_d{start_time:%Y%m%d_t%H%M%S}.geojson
-  - celcius:
-      file_pattern: AFIMG_{platform:s}_d{start_time:%Y%m%d_t%H%M%S}_celcius.geojson
-      unit: degC
-
-geojson-regional:
-  - si-units:
-      file_pattern: AFIMG_{platform:s}_d{start_time:%Y%m%d_t%H%M%S}_{region_name:s}.geojson
 
 """  # noqa
 
@@ -159,6 +147,9 @@
 
 TEST_ACTIVE_FIRES_FILEPATH = "./AFIMG_j01_d20210414_t1126439_e1128084_b17637_c20210414114130392094_cspp_dev.txt"
 TEST_ACTIVE_FIRES_FILEPATH2 = "./AFIMG_npp_d20230616_t1110054_e1111296_b60284_c20230616112418557033_cspp_dev.txt"
+TEST_ACTIVE_FIRES_FILEPATH3 = "./AFIMG_j01_d20230617_t1140564_e1142209_b28903_c20230617115513873196_cspp_dev.txt"
+
+TEST_ACTIVE_FIRES_FILEPATH4 = "./AFIMG_j01_d20230618_t0942269_e0943514_b28916_c20230618095604331171_cspp_dev.txt"
 
 
 TEST_ACTIVE_FIRES_FILE_DATA = """
@@ -222,6 +213,50 @@
 """
 
 
+# Here we have sorted out all detections not passing the filter mask!
+# So, 1 fire detection is left corresponding to what would end up in the geojson files:
+TEST_ACTIVE_FIRES_FILE_DATA3 = """
+# Active Fires I-band EDR
+#
+# source: AFIMG_j01_d20230617_t1140564_e1142209_b28903_c20230617115513873196_cspp_dev.nc
+# version: CSPP Active Fires version: cspp-active-fire-noaa_1.1.0
+#
+# column 1: latitude of fire pixel (degrees)
+# column 2: longitude of fire pixel (degrees)
+# column 3: I04 brightness temperature of fire pixel (K)
+# column 4: Along-scan fire pixel resolution (km)
+# column 5: Along-track fire pixel resolution (km)
+# column 6: detection confidence ([7,8,9]->[lo,med,hi])
+# column 7: fire radiative power (MW)
+#
+# number of fire pixels: 9
+#
+  64.46707153,   17.65028381,  330.15390015,  0.375,  0.375,    8,    3.75669074
+"""
+
+# Here we have sorted out all detections not passing the filter mask!
+# So, 2 fire detections are left corresponding to what would end up in the geojson files:
+TEST_ACTIVE_FIRES_FILE_DATA4 = """
+# Active Fires I-band EDR
+#
+# source: AFIMG_j01_d20230618_t0942269_e0943514_b28916_c20230618095604331171_cspp_dev.nc
+# version: CSPP Active Fires version: cspp-active-fire-noaa_1.1.0
+#
+# column 1: latitude of fire pixel (degrees)
+# column 2: longitude of fire pixel (degrees)
+# column 3: I04 brightness temperature of fire pixel (K)
+# column 4: Along-scan fire pixel resolution (km)
+# column 5: Along-track fire pixel resolution (km)
+# column 6: detection confidence ([7,8,9]->[lo,med,hi])
+# column 7: fire radiative power (MW)
+#
+# number of fire pixels: 10
+#
+  65.55922699,   17.62709618,  335.81488037,  0.375,  0.375,    8,    4.66374302
+  67.27209473,   20.14731216,  348.89843750,  0.375,  0.375,    8,   11.79477501
+"""
+
+
 @pytest.fixture
 def fake_active_fires_file_data():
     """Fake active fires output in a file - return an open stream with and the filepath."""
@@ -234,6 +269,36 @@ def fake_active_fires_file_data2():
     return io.StringIO(TEST_ACTIVE_FIRES_FILE_DATA2), TEST_ACTIVE_FIRES_FILEPATH2
 
 
+@pytest.fixture
+def fake_active_fires_ascii_file2(tmp_path):
+    """Create a fake active fires ascii file."""
+    file_path = tmp_path / TEST_ACTIVE_FIRES_FILEPATH2
+    with open(file_path, 'w') as fpt:
+        fpt.write(TEST_ACTIVE_FIRES_FILE_DATA2)
+
+    yield file_path
+
+
+@pytest.fixture
+def fake_active_fires_ascii_file3(tmp_path):
+    """Create a fake active fires ascii file."""
+    file_path = tmp_path / TEST_ACTIVE_FIRES_FILEPATH3
+    with open(file_path, 'w') as fpt:
+        fpt.write(TEST_ACTIVE_FIRES_FILE_DATA3)
+
+    yield file_path
+
+
+@pytest.fixture
+def fake_active_fires_ascii_file4(tmp_path):
+    """Create a fake active fires ascii file."""
+    file_path = tmp_path / TEST_ACTIVE_FIRES_FILEPATH4
+    with open(file_path, 'w') as fpt:
+        fpt.write(TEST_ACTIVE_FIRES_FILE_DATA4)
+
+    yield file_path
+
+
 @pytest.fixture
 def fake_token_file(tmp_path):
     """Write fake token file."""
@@ -244,6 +309,16 @@ def fake_token_file(tmp_path):
     yield file_path
 
 
+@pytest.fixture
+def fake_detection_id_cache_file(tmp_path):
+    """Write fake detection-id cache file."""
+    file_path = tmp_path / 'fire_detection_id_cache.txt'
+    with open(file_path, 'w') as fpt:
+        fpt.write('20230501-1')
+
+    yield file_path
+
+
 @pytest.fixture
 def fake_yamlconfig_file(tmp_path):
     """Write fake yaml config file."""
@@ -256,10 +331,21 @@ def fake_yamlconfig_file(tmp_path):
 
 @pytest.fixture
 def fake_yamlconfig_file_post_processing(tmp_path):
-    """Write fake yaml config file."""
+    """Write fake yaml config file - with no id cache file."""
+    file_path = tmp_path / 'test_af_post_processing_config.yaml'
+    with open(file_path, 'w') as fpt:
+        fpt.write(TEST_POST_PROCESSING_YAML_CONFIG_CONTENT)
+
+    yield file_path
+
+
+@pytest.fixture
+def fake_yamlconfig_file_post_processing_with_id_cache(tmp_path, fake_detection_id_cache_file):
+    """Write fake yaml config file - with a realistic id-cache file."""
     file_path = tmp_path / 'test_af_post_processing_config.yaml'
     with open(file_path, 'w') as fpt:
         fpt.write(TEST_POST_PROCESSING_YAML_CONFIG_CONTENT)
+        fpt.write('filepath_detection_id_cache: ' + str(fake_detection_id_cache_file))
 
     yield file_path