Merge c1fddd7 into b9fcb2e

README.md

@@ -104,3 +104,34 @@ feed = UsgsEarthquakeHazardsProgramFeed((21.3, -157.8), 'past_day_all_earthquake
                                         filter_radius=500, filter_minimum_magnitude=4.0)
 status, entries = feed.update()
 ```
+
+## Feed Managers
+
+The Feed Managers help managing feed updates over time, by notifying the 
+consumer of the feed about new feed entries, updates and removed entries 
+compared to the last feed update.
+
+* If the current feed update is the first one, then all feed entries will be 
+  reported as new. The feed manager will keep track of all feed entries' 
+  external IDs that it has successfully processed.
+* If the current feed update is not the first one, then the feed manager will 
+  produce three sets:
+  * Feed entries that were not in the previous feed update but are in the 
+    current feed update will be reported as new.
+  * Feed entries that were in the previous feed update and are still in the 
+    current feed update will be reported as to be updated.
+  * Feed entries that were in the previous feed update but are not in the 
+    current feed update will be reported to be removed.
+* If the current update fails, then all feed entries processed in the previous
+  feed update will be reported to be removed.
+
+After a successful update from the feed, the feed manager will provide two
+different dates:
+
+* `last_update` will be the timestamp of the last successful update from the
+  feed. This date may be useful if the consumer of this library wants to
+  treat intermittent errors from feed updates differently.
+* `last_timestamp` will be the latest timestamp extracted from the feed data. 
+  This requires that the underlying feed data actually contains a suitable 
+  date. This date may be useful if the consumer of this library wants to 
+  process feed entries differently if they haven't actually been updated.

geojson_client/__init__.py

@@ -3,6 +3,7 @@
 
 Fetches GeoJSON feed from URL to be defined by sub-class.
 """
+from datetime import datetime
 import geojson
 import logging
 
@@ -52,7 +53,8 @@ def update(self):
                     entries.append(self._new_entry(self._home_coordinates,
                                                    feature, global_data))
                 filtered_entries = self._filter_entries(entries)
-                self._last_timestamp = self._extract_last_timestamp(filtered_entries)
+                self._last_timestamp = self._extract_last_timestamp(
+                    filtered_entries)
                 return UPDATE_OK, filtered_entries
             else:
                 # Should not happen.
@@ -110,6 +112,11 @@ def _extract_last_timestamp(self, feed_entries):
         """Determine latest (newest) entry from the filtered feed."""
         return None
 
+    @property
+    def last_timestamp(self) -> Optional[datetime]:
+        """Return the last timestamp extracted from this feed."""
+        return self._last_timestamp
+
 
 class FeedEntry:
     """Feed entry base class."""

geojson_client/feed_manager.py

@@ -3,7 +3,9 @@
 
 This allows managing feeds and their entries throughout their life-cycle.
 """
+from datetime import datetime
 import logging
+from typing import Optional
 
 from geojson_client import UPDATE_OK, UPDATE_OK_NO_DATA
 
@@ -14,15 +16,15 @@ class FeedManagerBase:
     """Generic Feed manager."""
 
     def __init__(self, feed, generate_callback, update_callback,
-                 remove_callback, persistent_timestamp=False):
+                 remove_callback):
         """Initialise feed manager."""
         self._feed = feed
         self.feed_entries = {}
         self._managed_external_ids = set()
+        self._last_update = None
         self._generate_callback = generate_callback
         self._update_callback = update_callback
         self._remove_callback = remove_callback
-        self._persistent_timestamp = persistent_timestamp
 
     def __repr__(self):
         """Return string representation of this feed."""
@@ -37,6 +39,8 @@ def update(self):
             # Keep a copy of all feed entries for future lookups by entities.
             self.feed_entries = {entry.external_id: entry
                                  for entry in feed_entries}
+            # Record current time of update.
+            self._last_update = datetime.now()
             # For entity management the external ids from the feed are used.
             feed_external_ids = set(self.feed_entries)
             remove_external_ids = self._managed_external_ids.difference(
@@ -79,3 +83,13 @@ def _remove_entities(self, external_ids):
             _LOGGER.debug("Entity not current anymore %s", external_id)
             self._managed_external_ids.remove(external_id)
             self._remove_callback(external_id)
+
+    @property
+    def last_timestamp(self) -> Optional[datetime]:
+        """Return the last timestamp extracted from this feed."""
+        return self._feed.last_timestamp
+
+    @property
+    def last_update(self) -> Optional[datetime]:
+        """Return the last successful update of this feed."""
+        return self._last_update

geojson_client/nsw_rural_fire_service_feed.py

@@ -83,8 +83,9 @@ def _filter_entries(self, entries):
     def _extract_last_timestamp(self, feed_entries):
         """Determine latest (newest) entry from the filtered feed."""
         if feed_entries:
-            dates = sorted([entry.publication_date for entry in feed_entries],
-                           reverse=True)
+            dates = sorted(filter(
+                None, [entry.publication_date for entry in feed_entries]),
+                reverse=True)
             return dates[0]
         return None
 

tests/test_generic_feed.py

@@ -83,6 +83,7 @@ def test_update_error(self, mock_session, mock_request):
         feed = GenericFeed(home_coordinates, None)
         status, entries = feed.update()
         assert status == UPDATE_ERROR
+        self.assertIsNone(feed.last_timestamp)
 
     @mock.patch("requests.Request")
     @mock.patch("requests.Session")
@@ -147,6 +148,8 @@ def _remove_entity(external_id):
         entries = feed_manager.feed_entries
         self.assertIsNotNone(entries)
         assert len(entries) == 5
+        self.assertIsNotNone(feed_manager.last_update)
+
         assert len(generated_entity_external_ids) == 5
         assert len(updated_entity_external_ids) == 0
         assert len(removed_entity_external_ids) == 0

tests/test_nsw_rural_fire_service_feed.py

@@ -2,10 +2,12 @@
 import datetime
 import unittest
 from unittest import mock
+from unittest.mock import MagicMock
 
 from geojson_client import UPDATE_OK
 from geojson_client.nsw_rural_fire_service_feed import \
-    NswRuralFireServiceFeed, ATTRIBUTION, NswRuralFireServiceFeedManager
+    NswRuralFireServiceFeed, ATTRIBUTION, NswRuralFireServiceFeedManager, \
+    NswRuralFireServiceFeedEntry
 from tests.utils import load_fixture
 
 
@@ -33,6 +35,9 @@ def test_update_ok(self, mock_session, mock_request):
         assert status == UPDATE_OK
         self.assertIsNotNone(entries)
         assert len(entries) == 3
+        assert feed.last_timestamp \
+            == datetime.datetime(2018, 9, 21, 6, 40,
+                                 tzinfo=datetime.timezone.utc)
 
         feed_entry = entries[0]
         assert feed_entry.title == "Title 1"
@@ -126,6 +131,43 @@ def _remove_entity(external_id):
         entries = feed_manager.feed_entries
         self.assertIsNotNone(entries)
         assert len(entries) == 3
+        assert feed_manager.last_timestamp \
+            == datetime.datetime(2018, 9, 21, 6, 40,
+                                 tzinfo=datetime.timezone.utc)
         assert len(generated_entity_external_ids) == 3
         assert len(updated_entity_external_ids) == 0
         assert len(removed_entity_external_ids) == 0
+
+    def test_last_timestamp_empty(self):
+        """Test last timestamp."""
+        feed = NswRuralFireServiceFeed(None)
+
+        # Entries are None.
+        last_timestamp = feed._extract_last_timestamp(None)
+        self.assertIsNone(last_timestamp)
+
+        # Entries are empty.
+        last_timestamp = feed._extract_last_timestamp([])
+        self.assertIsNone(last_timestamp)
+
+        # Entries contain one with None date.
+        mock_entry_1 = MagicMock(spec=NswRuralFireServiceFeedEntry)
+        mock_entry_1.publication_date = None
+        datetime_1 = datetime.datetime(2019, 7, 4, 8, 0,
+                                       tzinfo=datetime.timezone.utc)
+        mock_entry_2 = MagicMock(spec=NswRuralFireServiceFeedEntry)
+        mock_entry_2.publication_date = datetime_1
+
+        last_timestamp = feed._extract_last_timestamp([mock_entry_1,
+                                                       mock_entry_2])
+        assert last_timestamp == datetime_1
+
+        # Entries contain multiple dates.
+        datetime_2 = datetime.datetime(2019, 7, 3, 8, 0,
+                                       tzinfo=datetime.timezone.utc)
+        mock_entry_3 = MagicMock(spec=NswRuralFireServiceFeedEntry)
+        mock_entry_3.publication_date = datetime_2
+        last_timestamp = feed._extract_last_timestamp([mock_entry_3,
+                                                       mock_entry_1,
+                                                       mock_entry_2])
+        assert last_timestamp == datetime_1

tox.ini

@@ -2,20 +2,25 @@
 envlist = py35, py36, py37, cov, cov_local
 
 [testenv]
-deps=pytest
+deps=
+    pytest
+    mock
 commands=pytest
 
 [testenv:cov]
 deps=
     pytest
     pytest-cov
+    mock
 commands=
     pytest --cov --cov-report= {posargs}
 
 [testenv:cov_local]
+basepython=python3.7
 deps=
     pytest
     pytest-cov
+    mock
 commands=
     pytest --cov --cov-report=
     coverage report