Merge branch 'master' into dev - v0.17.0-alpha2

Conflicts: CHANGES.rst pierky/arouteserver/version.py
pierky · Dec 12, 2017 · b28d0cb · b28d0cb
2 parents 8a06e01 + 343bee9
commit b28d0cb
Show file tree

Hide file tree

Showing 5 changed files with 21 additions and 5 deletions.
diff --git a/CHANGES.rst b/CHANGES.rst
@@ -10,6 +10,11 @@ next release
 
   AS-SETs can be prepended with an optional source: ``RIPE::AS-FOO``, ``RIPE::AS64496:AS-FOO``.
 
+v0.16.1
+-------
+
+- Fix: handle the new version of the JSON schema built by `arin-whois-bulk-parser <https://github.com/NLNOG/arin-whois-bulk-parser>`__.
+
 v0.16.0
 -------
 

diff --git a/pierky/arouteserver/arin_db_dump.py b/pierky/arouteserver/arin_db_dump.py
@@ -17,6 +17,7 @@
 import json
 import logging
 import os
+from packaging import version
 
 from six.moves.urllib.request import urlopen
 from six.moves.urllib.error import HTTPError
@@ -48,7 +49,7 @@ def load_data(self):
             json_schema = dic.get("json_schema", None)
             if json_schema is None:
                 raise ValueError("'json_schema' key is missing")
-            if json_schema != "0.0.2":
+            if version.parse(json_schema) >= version.parse("0.2"):
                 raise ValueError(
                     "unsupported JSON schema version: {}".format(json_schema)
                 )
@@ -106,10 +107,16 @@ def load_data(self):
                             )
                         )
         except ValueError as e:
-            raise ARINWhoisDBDumpError(
+            msg = (
                 "An error occurred while processing the ARIN Whois "
                 "database dump: {}".format(str(e))
             )
+            if self.from_cache:
+                logging.warning("{} - trying to bypass the cache".format(msg))
+                self.bypass_cache = True
+                self.load_data()
+            else:
+                raise ARINWhoisDBDumpError(msg)
 
     def _get_object_filename(self):
         return "arin-whois-db-dump.json"

diff --git a/pierky/arouteserver/cached_objects.py b/pierky/arouteserver/cached_objects.py
@@ -95,6 +95,8 @@ def __init__(self, **kwargs):
             self.cache_expiry_time = cache_expiry[self.EXPIRY_TIME_TAG]
 
         self.raw_data = None
+        self.bypass_cache = False
+        self.from_cache = False
 
     def _get_object_filename(self):
         raise NotImplementedError()
@@ -142,8 +144,9 @@ def _get_data(self):
         raise NotImplementedError()
 
     def load_data(self):
-        if self.load_data_from_cache():
+        if not self.bypass_cache and self.load_data_from_cache():
             logging.debug("Cache hit: {}".format(self._get_object_filepath()))
+            self.from_cache = True
             return
 
         # Children classes raise ExternalDataNoInfoError-derived exceptions
@@ -152,6 +155,7 @@ def load_data(self):
         # then the original exception is re-raised.
         try:
             self.raw_data = self._get_data()
+            self.from_cache = False
         except ExternalDataNoInfoError:
             self.save_data_to_cache()
             raise

diff --git a/pierky/arouteserver/version.py b/pierky/arouteserver/version.py
@@ -13,5 +13,5 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-__version__ = "0.17.0-alpha1" # pragma: no cover
+__version__ = "0.17.0-alpha2" # pragma: no cover
 COPYRIGHT_YEAR = 2017 # pragma: no cover
diff --git a/tests/static/arin_whois_db/dump.json b/tests/static/arin_whois_db/dump.json
@@ -1,5 +1,5 @@
 {
-  "json_schema": "0.0.2",
+  "json_schema": "0.1.0",
   "source": "ARIN-WHOIS",
   "whois_records": {
     "v4": [