Monitoring geoip2 (#286)

* use maxmind v2 db format if needed

* handle new geoip format properly

geonode/base/management/commands/updategeoip.py

@@ -22,6 +22,7 @@
 import os
 import logging
 import gzip
+import tarfile
 import urllib2 as urllib
 import traceback
 
@@ -33,7 +34,14 @@
 
 logger = logging.getLogger(__name__)
 
-URL = 'http://geolite.maxmind.com/download/geoip/database/GeoLiteCity.dat.gz'
+try:
+    from django.contrib.gis.geoip2 import GeoIP2 as GeoIP
+    URL = 'http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz'
+    OLD_FORMAT = False
+except ImportError:
+    from django.contrib.gis.geoip import GeoIP
+    URL = 'http://geolite.maxmind.com/download/geoip/database/GeoLiteCity.dat.gz'
+    OLD_FORMAT = True
 
 
 class Command(BaseCommand):
@@ -58,6 +66,37 @@ def handle(self, *args, **options):
         logger.info("Requesting %s", options['url'])
 
         r = urllib.urlopen(options['url'], timeout=10.0)
+
+        if OLD_FORMAT:
+            self.handle_old_format(r, fname)
+        else:
+            self.handle_new_format(r, fname)
+
+
+    def handle_new_format(self, r, fname):
+        try:
+            data = StringIO(r.read())
+            with tarfile.open(fileobj=data) as zfile:
+                members = zfile.getmembers()
+                for m in members:
+                    if m.name.endswith('GeoLite2-City.mmdb'):
+                        with open(fname, 'wb') as tofile:
+                            try:
+                                fromfile = zfile.extractfile(m)
+                                logger.info("Writing to %s", fname)
+                                tofile.write(fromfile.read())
+                            except Exception, err:
+                                logger.error("Cannot extract %s and write to %s: %s", m, fname, err, exc_info=err) 
+                                try:
+                                    os.remove(fname)
+                                except OSError:
+                                    logger.debug("Could not delete file %s", fname)
+                        return
+        except Exception, err:
+            logger.error("Cannot process %s: %s", r, err, exc_info=err)
+
+
+    def handle_old_format(self, r, fname):
         try:
             data = StringIO(r.read())
             with gzip.GzipFile(fileobj=data) as zfile: