Merge pull request #8 from adybbroe/fix-general-local2utc-time-conver…

…sions

Fix general local2utc time conversions

.github/workflows/ci.yaml

@@ -10,10 +10,10 @@ jobs:
       fail-fast: true
       matrix:
         os: ["ubuntu-latest", "macos-latest"]
-        python-version: ["3.8", "3.9"]
+        python-version: ["3.9", "3.10"]
         experimental: [false]
         include:
-          - python-version: "3.9"
+          - python-version: "3.10"
             os: "ubuntu-latest"
             experimental: true
 
@@ -42,7 +42,6 @@ jobs:
         shell: bash -l {0}
         run: |
           python -m pip install \
-          -f https://7933911d6844c6c53a7d-47bd50c35cd79bd838daf386af554a83.ssl.cf2.rackcdn.com \
           --no-deps --pre --upgrade \
           matplotlib \
           numpy \

activefires_pp/geometries_from_shapefiles.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-# Copyright (c) 2021 Adam.Dybbroe
+# Copyright (c) 2021, 2022 Adam.Dybbroe
 
 # Author(s):
 
@@ -56,7 +56,12 @@ def _get_proj(self):
         for filepath in self.filepaths:
             prj_filename = filepath.strip('.shp') + '.prj'
             crs = pycrs.load.from_file(prj_filename)
-            self.proj4str.append(crs.to_proj4())
+            if crs.name == 'SWEREF99_TM' and crs.proj.name.proj4 == 'utm':
+                utm_zone_proj4 = ' +zone=33'
+                proj4str = crs.to_proj4() + utm_zone_proj4
+            else:
+                proj4str = crs.to_proj4()
+            self.proj4str.append(proj4str)
 
         first_proj4_str = self.proj4str[0]
         for proj4_str in self.proj4str[1:]:
@@ -73,7 +78,7 @@ def _load_member_from_records(self, class_member, record_type):
 def _get_shapefile_paths(path, globstr='*.shp'):
     """Get full filepaths for all shapefiles in directory or simply return the paths as a list.
 
-    From a path to a directory with shapefiles or a full file path, 
+    From a path to a directory with shapefiles or a full file path,
     return list of file paths for all shapefiles.
     """
     if os.path.isfile(path):

activefires_pp/post_processing.py

@@ -1,11 +1,11 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-# Copyright (c) 2021 Adam.Dybbroe
+# Copyright (c) 2021 - 2022 Adam.Dybbroe
 
 # Author(s):
 
-#   Adam.Dybbroe <a000680@c21856.ad.smhi.se>
+#   Adam Dybbroe <Firstname.Lastname@smhi.se>
 
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -43,8 +43,8 @@
 from matplotlib.path import Path
 import shapely
 
-from activefires_pp.utils import datetime_from_utc_to_local
-from activefires_pp.utils import get_local_timezone
+from activefires_pp.utils import datetime_utc2local
+from activefires_pp.utils import get_local_timezone_offset
 from activefires_pp.utils import json_serial
 from activefires_pp.utils import read_config
 from activefires_pp.geometries_from_shapefiles import ShapeGeometry
@@ -71,7 +71,6 @@
 COL_NAMES = ["latitude", "longitude", "tb", "along_scan_res", "along_track_res", "conf", "power"]
 
 
-LOG_FORMAT = "[%(asctime)s %(levelname)-8s] %(message)s"
 logger = logging.getLogger(__name__)
 logging.getLogger("fiona").setLevel(logging.WARNING)
 
@@ -123,28 +122,27 @@ def _get_metadata_from_filename(self, infile_pattern):
 
     def _add_start_and_end_time_to_active_fires_data(self, localtime):
         """Add start and end time to active fires data."""
-        # Add start and end times:
         if localtime:
             logger.info("Convert to local time zone!")
-            starttime = datetime_from_utc_to_local(self.metadata['start_time'], self.timezone)
-            endtime = datetime_from_utc_to_local(self.metadata['end_time'], self.timezone)
-            self._afdata['starttime'] = self._apply_timezone_offset(starttime)
-            self._afdata['endtime'] = self._apply_timezone_offset(endtime)
+            starttime = datetime_utc2local(self.metadata['start_time'], self.timezone)
+            endtime = datetime_utc2local(self.metadata['end_time'], self.timezone)
         else:
-            starttime = self.metadata['start_time']
-            endtime = self.metadata['end_time']
-            self._afdata['starttime'] = np.repeat(starttime, len(self._afdata)).astype(np.datetime64)
-            self._afdata['endtime'] = np.repeat(endtime, len(self._afdata)).astype(np.datetime64)
+            starttime = datetime_utc2local(self.metadata['start_time'], 'GMT')
+            endtime = datetime_utc2local(self.metadata['end_time'], 'GMT')
+
+        starttime = starttime.replace(tzinfo=None)
+        endtime = endtime.replace(tzinfo=None)
+
+        self._afdata['starttime'] = np.repeat(starttime, len(self._afdata)).astype(np.datetime64)
+        self._afdata['endtime'] = np.repeat(endtime, len(self._afdata)).astype(np.datetime64)
 
         logger.info('Start and end times: %s %s',
                     str(self._afdata['starttime'][0]),
                     str(self._afdata['endtime'][0]))
 
     def _apply_timezone_offset(self, obstime):
         """Apply the time zone offset to the datetime objects."""
-        local_tz = get_local_timezone()
-        obstime_offset = local_tz.utcoffset(None)
-
+        obstime_offset = get_local_timezone_offset(self.timezone)
         return np.repeat(obstime.replace(tzinfo=None) + obstime_offset,
                          len(self._afdata)).astype(np.datetime64)
 
@@ -267,8 +265,10 @@ def store_geojson(output_filename, detections, platform_name=None):
         endtime = detections.iloc[idx].endtime
         mean_granule_time = starttime.to_pydatetime() + (endtime.to_pydatetime() -
                                                          starttime.to_pydatetime()) / 2.
+
         prop = {'power': detections.iloc[idx].power,
                 'tb': detections.iloc[idx].tb,
+                'confidence': int(detections.iloc[idx].conf),
                 'observation_time': json_serial(mean_granule_time)
                 }
         if platform_name:
@@ -287,8 +287,8 @@ def store_geojson(output_filename, detections, platform_name=None):
         logger.info("Create directory: %s", path)
         os.makedirs(path)
 
-    with open(output_filename, 'w') as f:
-        dump(feature_collection, f)
+    with open(output_filename, 'w') as fpt:
+        dump(feature_collection, fpt)
 
     return output_filename
 
@@ -373,7 +373,7 @@ def __init__(self, configfile, shp_boarders, shp_mask, regional_filtermask=None)
     def _setup_and_start_communication(self):
         """Set up the Posttroll communication and start the publisher."""
         logger.debug("Starting up... Input topic: %s", self.input_topic)
-        now = datetime_from_utc_to_local(datetime.now(), self.timezone)
+        now = datetime_utc2local(datetime.now(), self.timezone)
         logger.debug("Output times for timezone: {zone} Now = {time}".format(zone=str(self.timezone), time=now))
 
         self.listener = ListenerContainer(topics=[self.input_topic])

activefires_pp/tests/test_fires_filtering.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-# Copyright (c) 2021 Adam Dybbroe
+# Copyright (c) 2021, 2022 Adam Dybbroe
 
 # Author(s):
 

activefires_pp/tests/test_shapefile_geometries.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-# Copyright (c) 2021 Adam.Dybbroe
+# Copyright (c) 2021, 2022 Adam.Dybbroe
 
 # Author(s):
 
@@ -30,6 +30,8 @@
 
 TEST_CRS_PROJ = ('+proj=utm +ellps=GRS80 +a=6378137.0 +rf=298.257222101 +pm=0 +x_0=500000.0 ' +
                  '+y_0=0.0 +lon_0=15.0 +lat_0=0.0 +units=m +axis=enu +no_defs')
+TEST_CRS_PROJ_ZONE33 = ('+proj=utm +ellps=GRS80 +a=6378137.0 +rf=298.257222101 +pm=0 +x_0=500000.0 ' +
+                        '+y_0=0.0 +lon_0=15.0 +lat_0=0.0 +units=m +axis=enu +no_defs +zone=33')
 
 
 def fake_shapefiles_glob(dirname):
@@ -40,9 +42,21 @@ def fake_shapefiles_glob(dirname):
     return list_of_files
 
 
-class MyMockCrs(object):
+class MyMockProjName(object):
     def __init__(self):
+        self.proj4 = 'utm'
+
+
+class MyMockProj(object):
+    def __init__(self):
+        self.name = MyMockProjName()
+
+
+class MyMockCrs(object):
+    def __init__(self, crs_name='myname'):
         self._crs_proj = TEST_CRS_PROJ
+        self.name = crs_name
+        self.proj = MyMockProj()
 
     def to_proj4(self):
         return self._crs_proj
@@ -70,10 +84,10 @@ def test_shape_geometry_init_single_shapefile_path(load_from_file, get_shapefile
     mypath = '/my/shape/file/path/myshapefile.sph'
 
     get_shapefile_paths.return_value = [mypath]
-    load_from_file.return_value = MyMockCrs()
+    load_from_file.return_value = MyMockCrs('SWEREF99_TM')
     shpgeom = ShapeGeometry(mypath)
 
-    assert shpgeom.proj4str == TEST_CRS_PROJ
+    assert shpgeom.proj4str == TEST_CRS_PROJ_ZONE33
 
 
 @patch('activefires_pp.geometries_from_shapefiles._get_shapefile_paths')

activefires_pp/tests/test_utils.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-# Copyright (c) 2021 Adam.Dybbroe
+# Copyright (c) 2021, 2022 Adam.Dybbroe
 
 # Author(s):
 
@@ -24,12 +24,10 @@
 """
 
 import pytest
-import unittest
-from datetime import datetime
-
-from activefires_pp.utils import get_geometry_from_shapefile
-from activefires_pp.utils import datetime_from_utc_to_local
+from datetime import datetime, timedelta
+from activefires_pp.utils import datetime_utc2local
 from activefires_pp.utils import json_serial
+from freezegun import freeze_time
 
 
 def test_json_serial():
@@ -47,3 +45,25 @@ def test_json_serial():
     exception_raised = exec_info.value
 
     assert str(exception_raised) == "Type <class 'str'> not serializable"
+
+
+@freeze_time('2022-03-26 18:12:05')
+def test_utc2localtime_conversion():
+    """Test converting utc time to local time."""
+
+    atime1 = datetime.utcnow()
+    dtobj = datetime_utc2local(atime1, 'Europe/Stockholm')
+    assert dtobj.strftime('%Y%m%d-%H%M') == '20220326-1912'
+
+    atime2 = atime1 + timedelta(days=1)
+    dtobj = datetime_utc2local(atime2, 'Europe/Stockholm')
+    assert dtobj.strftime('%Y%m%d-%H%M') == '20220327-2012'
+
+    dtobj = datetime_utc2local(atime1, 'Australia/Sydney')
+    assert dtobj.strftime('%Y%m%d-%H%M') == '20220327-0512'
+
+    dtobj2 = datetime_utc2local(atime1, 'Etc/GMT-11')
+    assert dtobj2.strftime('%Y%m%d-%H%M') == '20220327-0512'
+
+    dtobj = datetime_utc2local(atime1 + timedelta(days=30), 'Australia/Sydney')
+    assert dtobj.strftime('%Y%m%d-%H%M') == '20220426-0412'

activefires_pp/utils.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-# Copyright (c) 2021 Adam.Dybbroe
+# Copyright (c) 2021, 2022 Adam.Dybbroe
 
 # Author(s):
 
@@ -23,9 +23,9 @@
 """
 """
 
-import pytz
 import cartopy.io.shapereader as shpreader
 from datetime import date, datetime, timezone, timedelta
+import zoneinfo
 import yaml
 from yaml import UnsafeLoader
 
@@ -38,21 +38,19 @@ def read_config(config_filepath):
     return config
 
 
-def datetime_from_utc_to_local(utc_dt, tzone):
-    """Convert datetime from UTC to local time."""
+def datetime_utc2local(utc_dtime, tzone_str, is_dst=True):
+    """Convert a UTC datetime to local time, using DST on default."""
+    tzone = zoneinfo.ZoneInfo(tzone_str)
+    utc_dtime = utc_dtime.replace(tzinfo=timezone.utc)
+    return utc_dtime.astimezone(tzone)
 
-    cest = pytz.timezone(tzone)
-    loc_dt = utc_dt.astimezone(cest)
 
-    return loc_dt
-
-
-def get_local_timezone():
-    """Get the local timezone of this computation environment.
-
-    https://stackoverflow.com/questions/2720319/python-figure-out-local-timezone
-    """
-    return datetime.now(timezone(timedelta(0))).astimezone().tzinfo
+def get_local_timezone_offset(timezone_str):
+    """Get the local time zone offset as a timedelta object."""
+    utcnow = datetime.utcnow()
+    utcnow = utcnow.replace(tzinfo=timezone.utc)
+    tzone = zoneinfo.ZoneInfo(timezone_str)
+    return utcnow.astimezone(tzone).replace(tzinfo=timezone.utc) - utcnow
 
 
 def get_geometry_from_shapefile(shapefile):

bin/active_fires_notifier.py

@@ -1,11 +1,11 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-# Copyright (c) 2021 Adam Dybbroe
+# Copyright (c) 2021-2022 Adam Dybbroe
 
 # Author(s):
 
-#   Adam Dybbroe <Firstname.Lastname at smhi.se>
+#   Adam Dybbroe <Firstname.Lastname@smhi.se>
 
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

bin/active_fires_postprocessing.py

@@ -1,11 +1,11 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-# Copyright (c) 2021 Adam.Dybbroe
+# Copyright (c) 2021-2022 Adam Dybbroe
 
 # Author(s):
 
-#   Adam.Dybbroe <a000680@c21856.ad.smhi.se>
+#   Adam Dybbroe <Firstname.Lastname@smhi.se>
 
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

continuous_integration/environment.yaml

@@ -20,6 +20,8 @@ dependencies:
   - cartopy
   - fiona
   - pandas
+  - freezegun
+  - openssl
   - pytz
   - pytest
   - pytest-cov

setup.cfg

@@ -5,7 +5,7 @@ setup_requires =
 
 [bdist_rpm]
 provides=activefires_pp
-requires=posttroll
+requires=posttroll trollsift pandas geojson fona matplotlib pycrs netifaces cartopy shapely
 release=1
 
 no-autoreq=True

setup.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-# Copyright (c) 2021 Adam Dybbroe
+# Copyright (c) 2021, 2022 Adam Dybbroe
 
 # Author(s):
 
@@ -40,9 +40,10 @@
 description = 'Post-processing of and notifications on Satellite active fire detections'
 
 requires = ['posttroll', 'netifaces', 'trollsift', 'setuptools_scm', 'pycrs',
-            'pytz', 'shapely', 'cartopy', 'pandas', 'geojson', 'fiona', 'matplotlib']
+            'shapely', 'cartopy', 'pandas', 'geojson', 'fiona', 'matplotlib']
 test_requires = ['mock', 'posttroll', 'trollsift', 'pycrs',
-                 'pytz', 'shapely', 'cartopy', 'pandas', 'geojson', 'fiona']
+                 'shapely', 'cartopy', 'pandas', 'geojson', 'fiona',
+                 'freezegun']
 
 setup(name="activefires-pp",
       description=description,
@@ -64,6 +65,6 @@
       zip_safe=False,
       install_requires=requires,
       tests_require=test_requires,
-      python_requires='>=3.8',
+      python_requires='>=3.9',
       use_scm_version=True
       )