diff --git a/activefires_pp/post_processing.py b/activefires_pp/post_processing.py index 6904db1..11bfdd5 100644 --- a/activefires_pp/post_processing.py +++ b/activefires_pp/post_processing.py @@ -615,9 +615,7 @@ def _initialize_fire_detection_id(self): def update_fire_detection_id(self): """Update the fire detection ID registry.""" now = datetime.utcnow() - tdelta = now - self._fire_detection_id['date'] - if (tdelta.total_seconds() > 24*3600 or - (tdelta.total_seconds() > 0 and self._fire_detection_id['date'].day != now.day)): + if self._fire_detection_id['date'].date() < now.date(): self._fire_detection_id = {'date': datetime.utcnow(), 'counter': 0} self._fire_detection_id['counter'] = self._fire_detection_id['counter'] + 1 diff --git a/activefires_pp/tests/conftest.py b/activefires_pp/tests/conftest.py index 73c8f9c..9968e43 100644 --- a/activefires_pp/tests/conftest.py +++ b/activefires_pp/tests/conftest.py @@ -63,8 +63,6 @@ output_dir: /path/where/the/filtered/results/will/be/stored -#filepath_detection_id_cache: /path/to/the/detection_id/cache/fire_detection_id_cache.txt - timezone: Europe/Stockholm @@ -149,6 +147,9 @@ TEST_ACTIVE_FIRES_FILEPATH = "./AFIMG_j01_d20210414_t1126439_e1128084_b17637_c20210414114130392094_cspp_dev.txt" TEST_ACTIVE_FIRES_FILEPATH2 = "./AFIMG_npp_d20230616_t1110054_e1111296_b60284_c20230616112418557033_cspp_dev.txt" +TEST_ACTIVE_FIRES_FILEPATH3 = "./AFIMG_j01_d20230617_t1140564_e1142209_b28903_c20230617115513873196_cspp_dev.txt" + +TEST_ACTIVE_FIRES_FILEPATH4 = "./AFIMG_j01_d20230618_t0942269_e0943514_b28916_c20230618095604331171_cspp_dev.txt" TEST_ACTIVE_FIRES_FILE_DATA = """ @@ -212,6 +213,50 @@ """ +# Here we have sorted out all detections not passing the filter mask! +# So, 1 fire detection is left corresponding to what would end up in the geojson files: +TEST_ACTIVE_FIRES_FILE_DATA3 = """ +# Active Fires I-band EDR +# +# source: AFIMG_j01_d20230617_t1140564_e1142209_b28903_c20230617115513873196_cspp_dev.nc +# version: CSPP Active Fires version: cspp-active-fire-noaa_1.1.0 +# +# column 1: latitude of fire pixel (degrees) +# column 2: longitude of fire pixel (degrees) +# column 3: I04 brightness temperature of fire pixel (K) +# column 4: Along-scan fire pixel resolution (km) +# column 5: Along-track fire pixel resolution (km) +# column 6: detection confidence ([7,8,9]->[lo,med,hi]) +# column 7: fire radiative power (MW) +# +# number of fire pixels: 9 +# + 64.46707153, 17.65028381, 330.15390015, 0.375, 0.375, 8, 3.75669074 +""" + +# Here we have sorted out all detections not passing the filter mask! +# So, 2 fire detections are left corresponding to what would end up in the geojson files: +TEST_ACTIVE_FIRES_FILE_DATA4 = """ +# Active Fires I-band EDR +# +# source: AFIMG_j01_d20230618_t0942269_e0943514_b28916_c20230618095604331171_cspp_dev.nc +# version: CSPP Active Fires version: cspp-active-fire-noaa_1.1.0 +# +# column 1: latitude of fire pixel (degrees) +# column 2: longitude of fire pixel (degrees) +# column 3: I04 brightness temperature of fire pixel (K) +# column 4: Along-scan fire pixel resolution (km) +# column 5: Along-track fire pixel resolution (km) +# column 6: detection confidence ([7,8,9]->[lo,med,hi]) +# column 7: fire radiative power (MW) +# +# number of fire pixels: 10 +# + 65.55922699, 17.62709618, 335.81488037, 0.375, 0.375, 8, 4.66374302 + 67.27209473, 20.14731216, 348.89843750, 0.375, 0.375, 8, 11.79477501 +""" + + @pytest.fixture def fake_active_fires_file_data(): """Fake active fires output in a file - return an open stream with and the filepath.""" @@ -224,6 +269,36 @@ def fake_active_fires_file_data2(): return io.StringIO(TEST_ACTIVE_FIRES_FILE_DATA2), TEST_ACTIVE_FIRES_FILEPATH2 +@pytest.fixture +def fake_active_fires_ascii_file2(tmp_path): + """Create a fake active fires ascii file.""" + file_path = tmp_path / TEST_ACTIVE_FIRES_FILEPATH2 + with open(file_path, 'w') as fpt: + fpt.write(TEST_ACTIVE_FIRES_FILE_DATA2) + + yield file_path + + +@pytest.fixture +def fake_active_fires_ascii_file3(tmp_path): + """Create a fake active fires ascii file.""" + file_path = tmp_path / TEST_ACTIVE_FIRES_FILEPATH3 + with open(file_path, 'w') as fpt: + fpt.write(TEST_ACTIVE_FIRES_FILE_DATA3) + + yield file_path + + +@pytest.fixture +def fake_active_fires_ascii_file4(tmp_path): + """Create a fake active fires ascii file.""" + file_path = tmp_path / TEST_ACTIVE_FIRES_FILEPATH4 + with open(file_path, 'w') as fpt: + fpt.write(TEST_ACTIVE_FIRES_FILE_DATA4) + + yield file_path + + @pytest.fixture def fake_token_file(tmp_path): """Write fake token file.""" diff --git a/activefires_pp/tests/test_fire_detection_id_handling.py b/activefires_pp/tests/test_fire_detection_id_handling.py index a721b51..9074c8b 100644 --- a/activefires_pp/tests/test_fire_detection_id_handling.py +++ b/activefires_pp/tests/test_fire_detection_id_handling.py @@ -23,90 +23,11 @@ """Test operations on the fire detection id.""" from unittest.mock import patch -from unittest import TestCase -import pandas as pd - -import io from datetime import datetime from freezegun import freeze_time from activefires_pp.post_processing import ActiveFiresShapefileFiltering from activefires_pp.post_processing import ActiveFiresPostprocessing -from activefires_pp.post_processing import COL_NAMES - - -TEST_ACTIVE_FIRES_FILEPATH2 = "./AFIMG_npp_d20230616_t1110054_e1111296_b60284_c20230616112418557033_cspp_dev.txt" -TEST_ACTIVE_FIRES_FILEPATH3 = "./AFIMG_j01_d20230617_t1140564_e1142209_b28903_c20230617115513873196_cspp_dev.txt" -TEST_ACTIVE_FIRES_FILEPATH4 = "./AFIMG_j01_d20230618_t0942269_e0943514_b28916_c20230618095604331171_cspp_dev.txt" - - -# Here we have sorted out all detections not passing the filter mask! -# So, 4 fire detections are left corresponding to what would end up in the geojson files: -TEST_ACTIVE_FIRES_FILE_DATA2 = """ -# Active Fires I-band EDR -# -# source: AFIMG_npp_d20230616_t1110054_e1111296_b60284_c20230616112418557033_cspp_dev.nc -# version: CSPP Active Fires version: cspp-active-fire-noaa_1.1.0 -# -# column 1: latitude of fire pixel (degrees) -# column 2: longitude of fire pixel (degrees) -# column 3: I04 brightness temperature of fire pixel (K) -# column 4: Along-scan fire pixel resolution (km) -# column 5: Along-track fire pixel resolution (km) -# column 6: detection confidence ([7,8,9]->[lo,med,hi]) -# column 7: fire radiative power (MW) -# -# number of fire pixels: 14 -# - 62.65801239, 17.25905228, 339.66326904, 0.375, 0.375, 8, 2.51202917 - 64.21694183, 17.42074966, 329.65161133, 0.375, 0.375, 8, 3.39806151 - 64.56904602, 16.60095215, 346.52050781, 0.375, 0.375, 8, 20.59289360 - 64.57222748, 16.59840012, 348.72860718, 0.375, 0.375, 8, 20.59289360 -""" - -# Here we have sorted out all detections not passing the filter mask! -# So, 1 fire detection is left corresponding to what would end up in the geojson files: -TEST_ACTIVE_FIRES_FILE_DATA3 = """ -# Active Fires I-band EDR -# -# source: AFIMG_j01_d20230617_t1140564_e1142209_b28903_c20230617115513873196_cspp_dev.nc -# version: CSPP Active Fires version: cspp-active-fire-noaa_1.1.0 -# -# column 1: latitude of fire pixel (degrees) -# column 2: longitude of fire pixel (degrees) -# column 3: I04 brightness temperature of fire pixel (K) -# column 4: Along-scan fire pixel resolution (km) -# column 5: Along-track fire pixel resolution (km) -# column 6: detection confidence ([7,8,9]->[lo,med,hi]) -# column 7: fire radiative power (MW) -# -# number of fire pixels: 9 -# - 64.46707153, 17.65028381, 330.15390015, 0.375, 0.375, 8, 3.75669074 -""" - - -# Here we have sorted out all detections not passing the filter mask! -# So, 2 fire detections are left corresponding to what would end up in the geojson files: -TEST_ACTIVE_FIRES_FILE_DATA4 = """ -# Active Fires I-band EDR -# -# source: AFIMG_j01_d20230618_t0942269_e0943514_b28916_c20230618095604331171_cspp_dev.nc -# version: CSPP Active Fires version: cspp-active-fire-noaa_1.1.0 -# -# column 1: latitude of fire pixel (degrees) -# column 2: longitude of fire pixel (degrees) -# column 3: I04 brightness temperature of fire pixel (K) -# column 4: Along-scan fire pixel resolution (km) -# column 5: Along-track fire pixel resolution (km) -# column 6: detection confidence ([7,8,9]->[lo,med,hi]) -# column 7: fire radiative power (MW) -# -# number of fire pixels: 10 -# - 65.55922699, 17.62709618, 335.81488037, 0.375, 0.375, 8, 4.66374302 - 67.27209473, 20.14731216, 348.89843750, 0.375, 0.375, 8, 11.79477501 -""" MY_FILE_PATTERN = ("AFIMG_{platform:s}_d{start_time:%Y%m%d_t%H%M%S%f}_e{end_hour:%H%M%S%f}_" + @@ -116,8 +37,8 @@ @freeze_time('2023-06-16 11:24:00') @patch('socket.gethostname') @patch('activefires_pp.post_processing.ActiveFiresPostprocessing._setup_and_start_communication') -@patch('activefires_pp.post_processing._read_data') -def test_add_unique_day_id_to_detections_sameday(readdata, setup_comm, gethostname, +def test_add_unique_day_id_to_detections_sameday(setup_comm, gethostname, + fake_active_fires_ascii_file2, fake_yamlconfig_file_post_processing): """Test adding unique id's to the fire detection data.""" gethostname.return_value = "my.host.name" @@ -127,34 +48,25 @@ def test_add_unique_day_id_to_detections_sameday(readdata, setup_comm, gethostna afpp = ActiveFiresPostprocessing(fake_yamlconfig_file_post_processing, myborders_file, mymask_file) - myfilepath = TEST_ACTIVE_FIRES_FILEPATH2 - - fstream = io.StringIO(TEST_ACTIVE_FIRES_FILE_DATA2) - afdata = pd.read_csv(fstream, index_col=None, header=None, comment='#', names=COL_NAMES) - readdata.return_value = afdata - this = ActiveFiresShapefileFiltering(filepath=myfilepath, timezone='GMT') - with patch('os.path.exists') as mypatch: - mypatch.return_value = True - afdata = this.get_af_data(filepattern=MY_FILE_PATTERN, localtime=False) + this = ActiveFiresShapefileFiltering(filepath=fake_active_fires_ascii_file2, timezone='GMT') + afdata = this.get_af_data(filepattern=MY_FILE_PATTERN, localtime=False) - TestCase().assertDictEqual(afpp._fire_detection_id, {'date': datetime.utcnow(), - 'counter': 0}) + assert afpp._fire_detection_id == {'date': datetime.utcnow(), 'counter': 0} # 4 fire detections, so (current) ID should be raised by 4 afdata = afpp.add_unique_day_id(afdata) assert 'detection_id' in afdata assert afdata['detection_id'].values.tolist() == ['20230616-1', '20230616-2', '20230616-3', '20230616-4'] - TestCase().assertDictEqual(afpp._fire_detection_id, {'date': datetime.utcnow(), - 'counter': 4}) + assert afpp._fire_detection_id == {'date': datetime.utcnow(), 'counter': 4} @freeze_time('2023-06-17 11:55:00') @patch('socket.gethostname') @patch('activefires_pp.post_processing.ActiveFiresPostprocessing._setup_and_start_communication') -@patch('activefires_pp.post_processing._read_data') -def test_add_unique_day_id_to_detections_24hours_plus(readdata, setup_comm, gethostname, +def test_add_unique_day_id_to_detections_24hours_plus(setup_comm, gethostname, + fake_active_fires_ascii_file3, fake_yamlconfig_file_post_processing): """Test adding unique id's to the fire detection data.""" gethostname.return_value = "my.host.name" @@ -166,33 +78,24 @@ def test_add_unique_day_id_to_detections_24hours_plus(readdata, setup_comm, geth myborders_file, mymask_file) afpp._fire_detection_id = {'date': datetime(2023, 6, 16, 11, 24, 0), 'counter': 4} - myfilepath = TEST_ACTIVE_FIRES_FILEPATH3 - - fstream = io.StringIO(TEST_ACTIVE_FIRES_FILE_DATA3) - afdata = pd.read_csv(fstream, index_col=None, header=None, comment='#', names=COL_NAMES) - readdata.return_value = afdata + this = ActiveFiresShapefileFiltering(filepath=fake_active_fires_ascii_file3, timezone='GMT') + afdata = this.get_af_data(filepattern=MY_FILE_PATTERN, localtime=False) - this = ActiveFiresShapefileFiltering(filepath=myfilepath, timezone='GMT') - with patch('os.path.exists') as mypatch: - mypatch.return_value = True - afdata = this.get_af_data(filepattern=MY_FILE_PATTERN, localtime=False) + assert afpp._fire_detection_id == {'date': datetime(2023, 6, 16, 11, 24, 0), 'counter': 4} - TestCase().assertDictEqual(afpp._fire_detection_id, {'date': datetime(2023, 6, 16, 11, 24, 0), - 'counter': 4}) # 1 new fire detection, so (current) ID should be raised - a new day, so id # starting over from 0, and a new date! afdata = afpp.add_unique_day_id(afdata) assert 'detection_id' in afdata assert afdata['detection_id'].values.tolist() == ['20230617-1'] - TestCase().assertDictEqual(afpp._fire_detection_id, {'date': datetime(2023, 6, 17, 11, 55, 0), - 'counter': 1}) + assert afpp._fire_detection_id, {'date': datetime(2023, 6, 17, 11, 55, 0), 'counter': 1} @freeze_time('2023-06-18 09:56:00') @patch('socket.gethostname') @patch('activefires_pp.post_processing.ActiveFiresPostprocessing._setup_and_start_communication') -@patch('activefires_pp.post_processing._read_data') -def test_add_unique_day_id_to_detections_newday_from_cache(readdata, setup_comm, gethostname, +def test_add_unique_day_id_to_detections_newday_from_cache(setup_comm, gethostname, + fake_active_fires_ascii_file4, fake_yamlconfig_file_post_processing_with_id_cache): """Test adding unique id's to the fire detection data.""" gethostname.return_value = "my.host.name" @@ -203,33 +106,24 @@ def test_add_unique_day_id_to_detections_newday_from_cache(readdata, setup_comm, afpp = ActiveFiresPostprocessing(fake_yamlconfig_file_post_processing_with_id_cache, myborders_file, mymask_file) - myfilepath = TEST_ACTIVE_FIRES_FILEPATH4 + this = ActiveFiresShapefileFiltering(filepath=fake_active_fires_ascii_file4, + timezone='GMT') + afdata = this.get_af_data(filepattern=MY_FILE_PATTERN, localtime=False) - fstream = io.StringIO(TEST_ACTIVE_FIRES_FILE_DATA4) - afdata = pd.read_csv(fstream, index_col=None, header=None, comment='#', names=COL_NAMES) - readdata.return_value = afdata - - this = ActiveFiresShapefileFiltering(filepath=myfilepath, timezone='GMT') - with patch('os.path.exists') as mypatch: - mypatch.return_value = True - afdata = this.get_af_data(filepattern=MY_FILE_PATTERN, localtime=False) - - TestCase().assertDictEqual(afpp._fire_detection_id, {'date': datetime(2023, 5, 1, 0, 0), - 'counter': 1}) + assert afpp._fire_detection_id == {'date': datetime(2023, 5, 1, 0, 0), 'counter': 1} # 2 new fire detections, so (current) ID should be raised - a new day, so id # starting over from 0, and a new date! afdata = afpp.add_unique_day_id(afdata) assert 'detection_id' in afdata assert afdata['detection_id'].values.tolist() == ['20230618-1', '20230618-2'] - TestCase().assertDictEqual(afpp._fire_detection_id, {'date': datetime(2023, 6, 18, 9, 56, 0), - 'counter': 2}) + assert afpp._fire_detection_id == {'date': datetime(2023, 6, 18, 9, 56, 0), 'counter': 2} @freeze_time('2023-06-18 09:56:00') @patch('socket.gethostname') @patch('activefires_pp.post_processing.ActiveFiresPostprocessing._setup_and_start_communication') -@patch('activefires_pp.post_processing._read_data') -def test_add_unique_day_id_to_detections_newday_no_cache(readdata, setup_comm, gethostname, +def test_add_unique_day_id_to_detections_newday_no_cache(setup_comm, gethostname, + fake_active_fires_ascii_file4, fake_yamlconfig_file_post_processing): """Test adding unique id's to the fire detection data.""" gethostname.return_value = "my.host.name" @@ -241,26 +135,17 @@ def test_add_unique_day_id_to_detections_newday_no_cache(readdata, setup_comm, g myborders_file, mymask_file) afpp._fire_detection_id = {'date': datetime(2023, 6, 17, 23, 55, 0), 'counter': 1} - myfilepath = TEST_ACTIVE_FIRES_FILEPATH4 - - fstream = io.StringIO(TEST_ACTIVE_FIRES_FILE_DATA4) - afdata = pd.read_csv(fstream, index_col=None, header=None, comment='#', names=COL_NAMES) - readdata.return_value = afdata - - this = ActiveFiresShapefileFiltering(filepath=myfilepath, timezone='GMT') - with patch('os.path.exists') as mypatch: - mypatch.return_value = True - afdata = this.get_af_data(filepattern=MY_FILE_PATTERN, localtime=False) + this = ActiveFiresShapefileFiltering(filepath=fake_active_fires_ascii_file4, + timezone='GMT') + afdata = this.get_af_data(filepattern=MY_FILE_PATTERN, localtime=False) - TestCase().assertDictEqual(afpp._fire_detection_id, {'date': datetime(2023, 6, 17, 23, 55), - 'counter': 1}) + assert afpp._fire_detection_id == {'date': datetime(2023, 6, 17, 23, 55), 'counter': 1} # 2 new fire detections, so (current) ID should be raised - a new day, so id # starting over from 0, and a new date! afdata = afpp.add_unique_day_id(afdata) assert 'detection_id' in afdata assert afdata['detection_id'].values.tolist() == ['20230618-1', '20230618-2'] - TestCase().assertDictEqual(afpp._fire_detection_id, {'date': datetime(2023, 6, 18, 9, 56, 0), - 'counter': 2}) + assert afpp._fire_detection_id == {'date': datetime(2023, 6, 18, 9, 56, 0), 'counter': 2} @patch('socket.gethostname') @@ -306,7 +191,7 @@ def test_initialize_fire_detection_id_nofile(readdata, setup_comm, gethostname, expected = {'date': datetime(2023, 6, 18, 12, 0, 0), 'counter': 0} afpp._initialize_fire_detection_id() - TestCase().assertDictEqual(afpp._fire_detection_id, expected) + assert afpp._fire_detection_id == expected @patch('socket.gethostname') @@ -329,7 +214,7 @@ def test_get_fire_detection_id_from_file(readdata, setup_comm, gethostname, tmp_ afpp.save_id_to_file() result = afpp.get_id_from_file() expected = {'date': datetime(2023, 6, 17), 'counter': 1} - TestCase().assertDictEqual(result, expected) + assert result == expected afpp._initialize_fire_detection_id() - TestCase().assertDictEqual(afpp._fire_detection_id, expected) + assert afpp._fire_detection_id == expected