Skip to content

Commit

Permalink
Less patching in tests and simplify code following review suggestions
Browse files Browse the repository at this point in the history 
Signed-off-by: Adam.Dybbroe <a000680@c21856.ad.smhi.se>
  • Loading branch information
Adam.Dybbroe committed Oct 2, 2023
1 parent 7d52279 commit 582f71c
Show file tree
Hide file tree
Showing 3 changed files with 107 additions and 149 deletions.
4 changes: 1 addition & 3 deletions activefires_pp/post_processing.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -615,9 +615,7 @@ def _initialize_fire_detection_id(self):
def update_fire_detection_id(self):
"""Update the fire detection ID registry."""
now = datetime.utcnow()
tdelta = now - self._fire_detection_id['date']
if (tdelta.total_seconds() > 24*3600 or
(tdelta.total_seconds() > 0 and self._fire_detection_id['date'].day != now.day)):
if self._fire_detection_id['date'].date() < now.date():
self._fire_detection_id = {'date': datetime.utcnow(), 'counter': 0}

self._fire_detection_id['counter'] = self._fire_detection_id['counter'] + 1
Expand Down
79 changes: 77 additions & 2 deletions activefires_pp/tests/conftest.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,8 +63,6 @@
output_dir: /path/where/the/filtered/results/will/be/stored
#filepath_detection_id_cache: /path/to/the/detection_id/cache/fire_detection_id_cache.txt
timezone: Europe/Stockholm
Expand Down Expand Up @@ -149,6 +147,9 @@

TEST_ACTIVE_FIRES_FILEPATH = "./AFIMG_j01_d20210414_t1126439_e1128084_b17637_c20210414114130392094_cspp_dev.txt"
TEST_ACTIVE_FIRES_FILEPATH2 = "./AFIMG_npp_d20230616_t1110054_e1111296_b60284_c20230616112418557033_cspp_dev.txt"
TEST_ACTIVE_FIRES_FILEPATH3 = "./AFIMG_j01_d20230617_t1140564_e1142209_b28903_c20230617115513873196_cspp_dev.txt"

TEST_ACTIVE_FIRES_FILEPATH4 = "./AFIMG_j01_d20230618_t0942269_e0943514_b28916_c20230618095604331171_cspp_dev.txt"


TEST_ACTIVE_FIRES_FILE_DATA = """
Expand Down Expand Up @@ -212,6 +213,50 @@
"""


# Here we have sorted out all detections not passing the filter mask!
# So, 1 fire detection is left corresponding to what would end up in the geojson files:
TEST_ACTIVE_FIRES_FILE_DATA3 = """
# Active Fires I-band EDR
#
# source: AFIMG_j01_d20230617_t1140564_e1142209_b28903_c20230617115513873196_cspp_dev.nc
# version: CSPP Active Fires version: cspp-active-fire-noaa_1.1.0
#
# column 1: latitude of fire pixel (degrees)
# column 2: longitude of fire pixel (degrees)
# column 3: I04 brightness temperature of fire pixel (K)
# column 4: Along-scan fire pixel resolution (km)
# column 5: Along-track fire pixel resolution (km)
# column 6: detection confidence ([7,8,9]->[lo,med,hi])
# column 7: fire radiative power (MW)
#
# number of fire pixels: 9
#
64.46707153, 17.65028381, 330.15390015, 0.375, 0.375, 8, 3.75669074
"""

# Here we have sorted out all detections not passing the filter mask!
# So, 2 fire detections are left corresponding to what would end up in the geojson files:
TEST_ACTIVE_FIRES_FILE_DATA4 = """
# Active Fires I-band EDR
#
# source: AFIMG_j01_d20230618_t0942269_e0943514_b28916_c20230618095604331171_cspp_dev.nc
# version: CSPP Active Fires version: cspp-active-fire-noaa_1.1.0
#
# column 1: latitude of fire pixel (degrees)
# column 2: longitude of fire pixel (degrees)
# column 3: I04 brightness temperature of fire pixel (K)
# column 4: Along-scan fire pixel resolution (km)
# column 5: Along-track fire pixel resolution (km)
# column 6: detection confidence ([7,8,9]->[lo,med,hi])
# column 7: fire radiative power (MW)
#
# number of fire pixels: 10
#
65.55922699, 17.62709618, 335.81488037, 0.375, 0.375, 8, 4.66374302
67.27209473, 20.14731216, 348.89843750, 0.375, 0.375, 8, 11.79477501
"""


@pytest.fixture
def fake_active_fires_file_data():
"""Fake active fires output in a file - return an open stream with and the filepath."""
Expand All @@ -224,6 +269,36 @@ def fake_active_fires_file_data2():
return io.StringIO(TEST_ACTIVE_FIRES_FILE_DATA2), TEST_ACTIVE_FIRES_FILEPATH2


@pytest.fixture
def fake_active_fires_ascii_file2(tmp_path):
"""Create a fake active fires ascii file."""
file_path = tmp_path / TEST_ACTIVE_FIRES_FILEPATH2
with open(file_path, 'w') as fpt:
fpt.write(TEST_ACTIVE_FIRES_FILE_DATA2)

yield file_path


@pytest.fixture
def fake_active_fires_ascii_file3(tmp_path):
"""Create a fake active fires ascii file."""
file_path = tmp_path / TEST_ACTIVE_FIRES_FILEPATH3
with open(file_path, 'w') as fpt:
fpt.write(TEST_ACTIVE_FIRES_FILE_DATA3)

yield file_path


@pytest.fixture
def fake_active_fires_ascii_file4(tmp_path):
"""Create a fake active fires ascii file."""
file_path = tmp_path / TEST_ACTIVE_FIRES_FILEPATH4
with open(file_path, 'w') as fpt:
fpt.write(TEST_ACTIVE_FIRES_FILE_DATA4)

yield file_path


@pytest.fixture
def fake_token_file(tmp_path):
"""Write fake token file."""
Expand Down
173 changes: 29 additions & 144 deletions activefires_pp/tests/test_fire_detection_id_handling.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,90 +23,11 @@
"""Test operations on the fire detection id."""

from unittest.mock import patch
from unittest import TestCase
import pandas as pd

import io
from datetime import datetime
from freezegun import freeze_time

from activefires_pp.post_processing import ActiveFiresShapefileFiltering
from activefires_pp.post_processing import ActiveFiresPostprocessing
from activefires_pp.post_processing import COL_NAMES


TEST_ACTIVE_FIRES_FILEPATH2 = "./AFIMG_npp_d20230616_t1110054_e1111296_b60284_c20230616112418557033_cspp_dev.txt"
TEST_ACTIVE_FIRES_FILEPATH3 = "./AFIMG_j01_d20230617_t1140564_e1142209_b28903_c20230617115513873196_cspp_dev.txt"
TEST_ACTIVE_FIRES_FILEPATH4 = "./AFIMG_j01_d20230618_t0942269_e0943514_b28916_c20230618095604331171_cspp_dev.txt"


# Here we have sorted out all detections not passing the filter mask!
# So, 4 fire detections are left corresponding to what would end up in the geojson files:
TEST_ACTIVE_FIRES_FILE_DATA2 = """
# Active Fires I-band EDR
#
# source: AFIMG_npp_d20230616_t1110054_e1111296_b60284_c20230616112418557033_cspp_dev.nc
# version: CSPP Active Fires version: cspp-active-fire-noaa_1.1.0
#
# column 1: latitude of fire pixel (degrees)
# column 2: longitude of fire pixel (degrees)
# column 3: I04 brightness temperature of fire pixel (K)
# column 4: Along-scan fire pixel resolution (km)
# column 5: Along-track fire pixel resolution (km)
# column 6: detection confidence ([7,8,9]->[lo,med,hi])
# column 7: fire radiative power (MW)
#
# number of fire pixels: 14
#
62.65801239, 17.25905228, 339.66326904, 0.375, 0.375, 8, 2.51202917
64.21694183, 17.42074966, 329.65161133, 0.375, 0.375, 8, 3.39806151
64.56904602, 16.60095215, 346.52050781, 0.375, 0.375, 8, 20.59289360
64.57222748, 16.59840012, 348.72860718, 0.375, 0.375, 8, 20.59289360
"""

# Here we have sorted out all detections not passing the filter mask!
# So, 1 fire detection is left corresponding to what would end up in the geojson files:
TEST_ACTIVE_FIRES_FILE_DATA3 = """
# Active Fires I-band EDR
#
# source: AFIMG_j01_d20230617_t1140564_e1142209_b28903_c20230617115513873196_cspp_dev.nc
# version: CSPP Active Fires version: cspp-active-fire-noaa_1.1.0
#
# column 1: latitude of fire pixel (degrees)
# column 2: longitude of fire pixel (degrees)
# column 3: I04 brightness temperature of fire pixel (K)
# column 4: Along-scan fire pixel resolution (km)
# column 5: Along-track fire pixel resolution (km)
# column 6: detection confidence ([7,8,9]->[lo,med,hi])
# column 7: fire radiative power (MW)
#
# number of fire pixels: 9
#
64.46707153, 17.65028381, 330.15390015, 0.375, 0.375, 8, 3.75669074
"""


# Here we have sorted out all detections not passing the filter mask!
# So, 2 fire detections are left corresponding to what would end up in the geojson files:
TEST_ACTIVE_FIRES_FILE_DATA4 = """
# Active Fires I-band EDR
#
# source: AFIMG_j01_d20230618_t0942269_e0943514_b28916_c20230618095604331171_cspp_dev.nc
# version: CSPP Active Fires version: cspp-active-fire-noaa_1.1.0
#
# column 1: latitude of fire pixel (degrees)
# column 2: longitude of fire pixel (degrees)
# column 3: I04 brightness temperature of fire pixel (K)
# column 4: Along-scan fire pixel resolution (km)
# column 5: Along-track fire pixel resolution (km)
# column 6: detection confidence ([7,8,9]->[lo,med,hi])
# column 7: fire radiative power (MW)
#
# number of fire pixels: 10
#
65.55922699, 17.62709618, 335.81488037, 0.375, 0.375, 8, 4.66374302
67.27209473, 20.14731216, 348.89843750, 0.375, 0.375, 8, 11.79477501
"""


MY_FILE_PATTERN = ("AFIMG_{platform:s}_d{start_time:%Y%m%d_t%H%M%S%f}_e{end_hour:%H%M%S%f}_" +
Expand All @@ -116,8 +37,8 @@
@freeze_time('2023-06-16 11:24:00')
@patch('socket.gethostname')
@patch('activefires_pp.post_processing.ActiveFiresPostprocessing._setup_and_start_communication')
@patch('activefires_pp.post_processing._read_data')
def test_add_unique_day_id_to_detections_sameday(readdata, setup_comm, gethostname,
def test_add_unique_day_id_to_detections_sameday(setup_comm, gethostname,
fake_active_fires_ascii_file2,
fake_yamlconfig_file_post_processing):
"""Test adding unique id's to the fire detection data."""
gethostname.return_value = "my.host.name"
Expand All @@ -127,34 +48,25 @@ def test_add_unique_day_id_to_detections_sameday(readdata, setup_comm, gethostna

afpp = ActiveFiresPostprocessing(fake_yamlconfig_file_post_processing,
myborders_file, mymask_file)
myfilepath = TEST_ACTIVE_FIRES_FILEPATH2

fstream = io.StringIO(TEST_ACTIVE_FIRES_FILE_DATA2)
afdata = pd.read_csv(fstream, index_col=None, header=None, comment='#', names=COL_NAMES)
readdata.return_value = afdata

this = ActiveFiresShapefileFiltering(filepath=myfilepath, timezone='GMT')
with patch('os.path.exists') as mypatch:
mypatch.return_value = True
afdata = this.get_af_data(filepattern=MY_FILE_PATTERN, localtime=False)
this = ActiveFiresShapefileFiltering(filepath=fake_active_fires_ascii_file2, timezone='GMT')
afdata = this.get_af_data(filepattern=MY_FILE_PATTERN, localtime=False)

TestCase().assertDictEqual(afpp._fire_detection_id, {'date': datetime.utcnow(),
'counter': 0})
assert afpp._fire_detection_id == {'date': datetime.utcnow(), 'counter': 0}

# 4 fire detections, so (current) ID should be raised by 4
afdata = afpp.add_unique_day_id(afdata)
assert 'detection_id' in afdata
assert afdata['detection_id'].values.tolist() == ['20230616-1', '20230616-2',
'20230616-3', '20230616-4']
TestCase().assertDictEqual(afpp._fire_detection_id, {'date': datetime.utcnow(),
'counter': 4})
assert afpp._fire_detection_id == {'date': datetime.utcnow(), 'counter': 4}


@freeze_time('2023-06-17 11:55:00')
@patch('socket.gethostname')
@patch('activefires_pp.post_processing.ActiveFiresPostprocessing._setup_and_start_communication')
@patch('activefires_pp.post_processing._read_data')
def test_add_unique_day_id_to_detections_24hours_plus(readdata, setup_comm, gethostname,
def test_add_unique_day_id_to_detections_24hours_plus(setup_comm, gethostname,
fake_active_fires_ascii_file3,
fake_yamlconfig_file_post_processing):
"""Test adding unique id's to the fire detection data."""
gethostname.return_value = "my.host.name"
Expand All @@ -166,33 +78,24 @@ def test_add_unique_day_id_to_detections_24hours_plus(readdata, setup_comm, geth
myborders_file, mymask_file)
afpp._fire_detection_id = {'date': datetime(2023, 6, 16, 11, 24, 0), 'counter': 4}

myfilepath = TEST_ACTIVE_FIRES_FILEPATH3

fstream = io.StringIO(TEST_ACTIVE_FIRES_FILE_DATA3)
afdata = pd.read_csv(fstream, index_col=None, header=None, comment='#', names=COL_NAMES)
readdata.return_value = afdata
this = ActiveFiresShapefileFiltering(filepath=fake_active_fires_ascii_file3, timezone='GMT')
afdata = this.get_af_data(filepattern=MY_FILE_PATTERN, localtime=False)

this = ActiveFiresShapefileFiltering(filepath=myfilepath, timezone='GMT')
with patch('os.path.exists') as mypatch:
mypatch.return_value = True
afdata = this.get_af_data(filepattern=MY_FILE_PATTERN, localtime=False)
assert afpp._fire_detection_id == {'date': datetime(2023, 6, 16, 11, 24, 0), 'counter': 4}

TestCase().assertDictEqual(afpp._fire_detection_id, {'date': datetime(2023, 6, 16, 11, 24, 0),
'counter': 4})
# 1 new fire detection, so (current) ID should be raised - a new day, so id
# starting over from 0, and a new date!
afdata = afpp.add_unique_day_id(afdata)
assert 'detection_id' in afdata
assert afdata['detection_id'].values.tolist() == ['20230617-1']
TestCase().assertDictEqual(afpp._fire_detection_id, {'date': datetime(2023, 6, 17, 11, 55, 0),
'counter': 1})
assert afpp._fire_detection_id, {'date': datetime(2023, 6, 17, 11, 55, 0), 'counter': 1}


@freeze_time('2023-06-18 09:56:00')
@patch('socket.gethostname')
@patch('activefires_pp.post_processing.ActiveFiresPostprocessing._setup_and_start_communication')
@patch('activefires_pp.post_processing._read_data')
def test_add_unique_day_id_to_detections_newday_from_cache(readdata, setup_comm, gethostname,
def test_add_unique_day_id_to_detections_newday_from_cache(setup_comm, gethostname,
fake_active_fires_ascii_file4,
fake_yamlconfig_file_post_processing_with_id_cache):
"""Test adding unique id's to the fire detection data."""
gethostname.return_value = "my.host.name"
Expand All @@ -203,33 +106,24 @@ def test_add_unique_day_id_to_detections_newday_from_cache(readdata, setup_comm,
afpp = ActiveFiresPostprocessing(fake_yamlconfig_file_post_processing_with_id_cache,
myborders_file, mymask_file)

myfilepath = TEST_ACTIVE_FIRES_FILEPATH4
this = ActiveFiresShapefileFiltering(filepath=fake_active_fires_ascii_file4,
timezone='GMT')
afdata = this.get_af_data(filepattern=MY_FILE_PATTERN, localtime=False)

fstream = io.StringIO(TEST_ACTIVE_FIRES_FILE_DATA4)
afdata = pd.read_csv(fstream, index_col=None, header=None, comment='#', names=COL_NAMES)
readdata.return_value = afdata

this = ActiveFiresShapefileFiltering(filepath=myfilepath, timezone='GMT')
with patch('os.path.exists') as mypatch:
mypatch.return_value = True
afdata = this.get_af_data(filepattern=MY_FILE_PATTERN, localtime=False)

TestCase().assertDictEqual(afpp._fire_detection_id, {'date': datetime(2023, 5, 1, 0, 0),
'counter': 1})
assert afpp._fire_detection_id == {'date': datetime(2023, 5, 1, 0, 0), 'counter': 1}
# 2 new fire detections, so (current) ID should be raised - a new day, so id
# starting over from 0, and a new date!
afdata = afpp.add_unique_day_id(afdata)
assert 'detection_id' in afdata
assert afdata['detection_id'].values.tolist() == ['20230618-1', '20230618-2']
TestCase().assertDictEqual(afpp._fire_detection_id, {'date': datetime(2023, 6, 18, 9, 56, 0),
'counter': 2})
assert afpp._fire_detection_id == {'date': datetime(2023, 6, 18, 9, 56, 0), 'counter': 2}


@freeze_time('2023-06-18 09:56:00')
@patch('socket.gethostname')
@patch('activefires_pp.post_processing.ActiveFiresPostprocessing._setup_and_start_communication')
@patch('activefires_pp.post_processing._read_data')
def test_add_unique_day_id_to_detections_newday_no_cache(readdata, setup_comm, gethostname,
def test_add_unique_day_id_to_detections_newday_no_cache(setup_comm, gethostname,
fake_active_fires_ascii_file4,
fake_yamlconfig_file_post_processing):
"""Test adding unique id's to the fire detection data."""
gethostname.return_value = "my.host.name"
Expand All @@ -241,26 +135,17 @@ def test_add_unique_day_id_to_detections_newday_no_cache(readdata, setup_comm, g
myborders_file, mymask_file)
afpp._fire_detection_id = {'date': datetime(2023, 6, 17, 23, 55, 0), 'counter': 1}

myfilepath = TEST_ACTIVE_FIRES_FILEPATH4

fstream = io.StringIO(TEST_ACTIVE_FIRES_FILE_DATA4)
afdata = pd.read_csv(fstream, index_col=None, header=None, comment='#', names=COL_NAMES)
readdata.return_value = afdata

this = ActiveFiresShapefileFiltering(filepath=myfilepath, timezone='GMT')
with patch('os.path.exists') as mypatch:
mypatch.return_value = True
afdata = this.get_af_data(filepattern=MY_FILE_PATTERN, localtime=False)
this = ActiveFiresShapefileFiltering(filepath=fake_active_fires_ascii_file4,
timezone='GMT')
afdata = this.get_af_data(filepattern=MY_FILE_PATTERN, localtime=False)

TestCase().assertDictEqual(afpp._fire_detection_id, {'date': datetime(2023, 6, 17, 23, 55),
'counter': 1})
assert afpp._fire_detection_id == {'date': datetime(2023, 6, 17, 23, 55), 'counter': 1}
# 2 new fire detections, so (current) ID should be raised - a new day, so id
# starting over from 0, and a new date!
afdata = afpp.add_unique_day_id(afdata)
assert 'detection_id' in afdata
assert afdata['detection_id'].values.tolist() == ['20230618-1', '20230618-2']
TestCase().assertDictEqual(afpp._fire_detection_id, {'date': datetime(2023, 6, 18, 9, 56, 0),
'counter': 2})
assert afpp._fire_detection_id == {'date': datetime(2023, 6, 18, 9, 56, 0), 'counter': 2}


@patch('socket.gethostname')
Expand Down Expand Up @@ -306,7 +191,7 @@ def test_initialize_fire_detection_id_nofile(readdata, setup_comm, gethostname,
expected = {'date': datetime(2023, 6, 18, 12, 0, 0), 'counter': 0}

afpp._initialize_fire_detection_id()
TestCase().assertDictEqual(afpp._fire_detection_id, expected)
assert afpp._fire_detection_id == expected


@patch('socket.gethostname')
Expand All @@ -329,7 +214,7 @@ def test_get_fire_detection_id_from_file(readdata, setup_comm, gethostname, tmp_
afpp.save_id_to_file()
result = afpp.get_id_from_file()
expected = {'date': datetime(2023, 6, 17), 'counter': 1}
TestCase().assertDictEqual(result, expected)
assert result == expected

afpp._initialize_fire_detection_id()
TestCase().assertDictEqual(afpp._fire_detection_id, expected)
assert afpp._fire_detection_id == expected

0 comments on commit 582f71c

Please sign in to comment.