Merge e9e4021 into 33a20c2

activefires_pp/post_processing.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-# Copyright (c) 2021 - 2023 Adam.Dybbro
+# Copyright (c) 2021 - 2023 Adam.Dybbroe
 
 # Author(s):
 
@@ -43,6 +43,7 @@
 import shapely
 
 from activefires_pp.utils import datetime_utc2local
+from activefires_pp.utils import UnitConverter
 from activefires_pp.utils import get_local_timezone_offset
 from activefires_pp.utils import json_serial
 from activefires_pp.config import read_config
@@ -270,9 +271,20 @@ def geojson_feature_collection_from_detections(detections, platform_name=None):
         prop = {'power': detections.iloc[idx].power,
                 'tb': detections.iloc[idx].tb,
                 'confidence': int(detections.iloc[idx].conf),
-                'id': detections.iloc[idx].detection_id,
                 'observation_time': json_serial(mean_granule_time)
                 }
+
+        try:
+            prop['tb_celcius'] = detections.iloc[idx].tb_celcius
+        except AttributeError:
+            logger.debug("Failed adding the TB in celcius!")
+            pass
+        try:
+            prop['id'] = detections.iloc[idx].detection_id
+        except AttributeError:
+            logger.debug("Failed adding the unique detection id!")
+            pass
+
         if platform_name:
             prop['platform_name'] = platform_name
         else:
@@ -356,8 +368,14 @@ def __init__(self, configfile, shp_borders, shp_mask, regional_filtermask=None):
         self.input_topic = self.options['subscribe_topics'][0]
         self.output_topic = self.options['publish_topic']
         self.infile_pattern = self.options.get('af_pattern_ibands')
+
         self.outfile_pattern_national = self.options.get('geojson_file_pattern_national')
         self.outfile_pattern_regional = self.options.get('geojson_file_pattern_regional')
+
+        # self.regional_outputs = self.options.get('geojson-regional')
+        # self.national_outputs = self.options.get('geojson-national')
+        # self.set_output_filename_parsers()
+
         self.output_dir = self.options.get('output_dir', '/tmp')
         self.filepath_detection_id_cache = self.options.get('filepath_detection_id_cache')
 
@@ -378,6 +396,11 @@ def _setup_and_start_communication(self):
         now = datetime_utc2local(datetime.now(), self.timezone)
         logger.debug("Output times for timezone: {zone} Now = {time}".format(zone=str(self.timezone), time=now))
 
+        tic = time.time()
+        units = {'temperature': 'degC'}
+        self.unit_converter = UnitConverter(units)
+        logger.debug("Unit conversion initialization with Pint took %f seconds.", tic - time.time())
+
         self._check_borders_shapes_exists()
 
         self.listener = ListenerContainer(topics=[self.input_topic])
@@ -450,10 +473,11 @@ def do_postprocessing_on_message(self, msg, filename):
             logger.debug("No fires - so no regional filtering to be done!")
             return
 
-        # It is here that we should add a uniue day-ID to each of the detections!
         afdata = self.add_unique_day_id(afdata)
         self.save_id_to_file()
 
+        afdata = self.add_tb_celcius(afdata)
+
         # 1) Create geojson feature collection
         # 2) Dump geojson data to disk
         feature_collection = geojson_feature_collection_from_detections(afdata,
@@ -531,7 +555,6 @@ def regional_fires_filtering_and_publishing(self, msg, regional_fmask, afsff_obj
             logger.debug("Output file path = %s", out_filepath)
             data_in_region = afdata[regional_fmask[region_name]['mask']]
 
-            # filepath = store_geojson(out_filepath, data_in_region, platform_name=fmda['platform'])
             feature_collection = geojson_feature_collection_from_detections(data_in_region,
                                                                             platform_name=fmda['platform'])
             if feature_collection is None:
@@ -579,6 +602,31 @@ def fires_filtering(self, msg, af_shapeff):
 
         return afdata_ff
 
+    # def create_output(self, data, metadata, outputs):
+    #     """Create geojson output and return filepaths."""
+    #     paths_and_units = []
+    #     for item in outputs:
+    #         for output in item:
+    #             filepath = os.path.join(self.output_dir, item[output]['parser'].compose(metadata))
+    #             if 'unit' in item[output]:
+    #                 paths_and_units.append({'filepath': filepath, 'unit': item[output]['unit']})
+    #             else:
+    #                 paths_and_units.append({'filepath': filepath})
+
+    #     filepaths = []
+    #     for item in paths_and_units:
+    #         out_filepath = item['filepath']
+    #         logger.debug("Output file path = %s", out_filepath)
+    #         if 'unit' in item:
+    #             filepath = store_geojson(out_filepath, data, platform_name=metadata['platform'],
+    #                                      units={'temperature': item['unit']})
+    #         else:
+    #             filepath = store_geojson(out_filepath, data, platform_name=metadata['platform'])
+
+    #         filepaths.append(filepath)
+
+    #     return filepaths
+
     def get_output_messages(self, filepath, msg, number_of_data):
         """Generate the adequate output message(s) depending on if an output file was created or not."""
         logger.info("Geojson file created! Number of fires = %d", number_of_data)
@@ -670,6 +718,16 @@ def add_unique_day_id(self, afdata):
         afdata['detection_id'] = id_list
         return afdata
 
+    def add_tb_celcius(self, data_frame):
+        """Add a column with TB in Celcius to the fire detection data frame."""
+        tbc_list = []
+        for _i in range(len(data_frame)):
+            tbc = self.unit_converter.convert('temperature', data_frame['tb'].iloc[_i])
+            tbc_list.append(tbc.magnitude)
+
+        data_frame['tb_celcius'] = tbc_list
+        return data_frame
+
     def close(self):
         """Shutdown the Active Fires postprocessing."""
         logger.info('Terminating Active Fires post processing.')

activefires_pp/tests/conftest.py

@@ -47,14 +47,26 @@
 
 af_pattern_ibands: AFIMG_{platform:s}_d{start_time:%Y%m%d_t%H%M%S%f}_e{end_hour:%H%M%S%f}_b{orbit:s}_c{processing_time:%Y%m%d%H%M%S%f}_cspp_dev.txt
 
-geojson_file_pattern_national: AFIMG_{platform:s}_d{start_time:%Y%m%d_t%H%M%S}.geojson
-geojson_file_pattern_regional: AFIMG_{platform:s}_d{start_time:%Y%m%d_t%H%M%S}_{region_name:s}.geojson
+#geojson_file_pattern_national: AFIMG_{platform:s}_d{start_time:%Y%m%d_t%H%M%S}.geojson
+#geojson_file_pattern_regional: AFIMG_{platform:s}_d{start_time:%Y%m%d_t%H%M%S}_{region_name:s}.geojson
 
 regional_shapefiles_format: omr_{region_code:s}_Buffer.{ext:s}
 
 output_dir: /path/where/the/filtered/results/will/be/stored
 
 timezone: Europe/Stockholm
+
+geojson-national:
+  - kelvin:
+      file_pattern: AFIMG_{platform:s}_d{start_time:%Y%m%d_t%H%M%S}.geojson
+  - celcius:
+      file_pattern: AFIMG_{platform:s}_d{start_time:%Y%m%d_t%H%M%S}_celcius.geojson
+      unit: degC
+
+geojson-regional:
+  - si-units:
+      file_pattern: AFIMG_{platform:s}_d{start_time:%Y%m%d_t%H%M%S}_{region_name:s}.geojson
+
 """  # noqa
 
 TEST_YAML_TOKENS = """xauth_tokens:

activefires_pp/tests/test_config.py

@@ -26,8 +26,8 @@
 from activefires_pp.config import get_xauthentication_token
 
 
-def test_get_yaml_configuration(fake_yamlconfig_file):
-    """Test read and get the yaml configuration from file."""
+def test_get_yaml_configuration_for_alarm_filtering(fake_yamlconfig_file):
+    """Test read and get the yaml configuration from file for alarm filtering."""
     config = read_config(fake_yamlconfig_file)
     assert config['subscribe_topics'] == '/VIIRS/L2/Fires/PP/National'
     assert config['publish_topic'] == '/VIIRS/L2/Fires/PP/SOSAlarm'
@@ -40,3 +40,24 @@ def test_get_xauthentication_token(fake_token_file):
     """Test getting the xauthentication token from a file."""
     fake_token = get_xauthentication_token(fake_token_file)
     assert fake_token == 'my-token'
+
+
+def test_read_yaml_configuration_for_postprocessing(fake_yamlconfig_file_post_processing):
+    """Test read in the yaml configuration for fires post processing."""
+    config = read_config(fake_yamlconfig_file_post_processing)
+
+    assert config['subscribe_topics'] == 'VIIRS/L2/AFI'
+    assert config['publish_topic'] == '/VIIRS/L2/Fires/PP'
+    assert config['timezone'] == 'Europe/Stockholm'
+    assert config['af_pattern_ibands'] == 'AFIMG_{platform:s}_d{start_time:%Y%m%d_t%H%M%S%f}_e{end_hour:%H%M%S%f}_b{orbit:s}_c{processing_time:%Y%m%d%H%M%S%f}_cspp_dev.txt'  # noqa
+    assert config['regional_shapefiles_format'] == 'omr_{region_code:s}_Buffer.{ext:s}'
+    assert config['output_dir'] == '/path/where/the/filtered/results/will/be/stored'
+    assert len(config['geojson-national']) == 2
+    assert config['geojson-national'][1] == {'celcius':
+                                             {'file_pattern':
+                                              'AFIMG_{platform:s}_d{start_time:%Y%m%d_t%H%M%S}_celcius.geojson',  # noqa
+                                              'unit': 'degC'}}
+    assert len(config['geojson-regional']) == 1
+    assert config['geojson-regional'][0] == {'si-units':
+                                             {'file_pattern':
+                                              'AFIMG_{platform:s}_d{start_time:%Y%m%d_t%H%M%S}_{region_name:s}.geojson'}}  # noqa