Added comments and docs pointers

Added fetch info step. (Note, may require manual action)
Wrapped data not found with handler. Fixed decl. race condition
flake8 fixes

Makefile

@@ -6,10 +6,31 @@ NO = bin/nosetests -s --with-xunit
 
 all: build
 
+# Note:
+# Unfortunately, this may fail with a 503 error. If that's the case, you
+# will have to go to the download page directly.
+# You will need both the City and Country databases.
+fetch: data/GeoIP.dat data/GeoLiteCity.dat
+	@echo "Fetching data"
+
+data/GeoIP.dat:
+	mkdir -p data
+	wget -P data "http://geolite.maxmind.com/download/geoip/database/GeoLiteCountry/GeoIP.dat.gz"
+	gzip -d data/GeoIP.dat.gz
+	cd -
+
+data/GeoLiteCity.dat:
+	mkdir -p data
+	cd data
+	wget -P data "http://geolite.maxmind.com/download/geoip/database/GeoLiteCity.dat.gz"
+	gzip -d data/GeoLiteCity.dat
+	cd -
+
 build:
 	$(VE) --no-site-packages .
 	$(PI) install -r prod-reqs.txt
 	$(PY) setup.py build
+	@echo "Run 'make fetch' to fetch GeoIP data."
 
 test:
 	$(NO) $(APPNAME)

geoip/__init__.py

@@ -1,17 +1,23 @@
 """
+    This Source Code Form is subject to the terms of the Mozilla Public
+    License, v. 2.0. If a copy of the MPL was not distributed with this
+    file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
     Stand alone MaxMind GeoIP binary service
 
     In order to isloate potential segfaults from the C API, creating this as
     a pollable service that can be quickly restarted.
 
-    based off of django_geoip.
+    based off of django_geoip
+    <https://github.com/futurecolors/django-geoip/>
 
 """
 
 from __future__ import absolute_import
 
 try:
-    from .base import GeoIP, GeoIPException
+    from base import GeoIP
     HAS_GEOIP = True
 except:
     HAS_GEOIP = False

geoip/base.py

@@ -1,7 +1,6 @@
 import os
 import re
 from ctypes import c_char_p
-from libgeoip import GeoIPException
 
 #from django.core.validators import ipv4_re
 
@@ -24,6 +23,8 @@
 
 #### GeoIP classes ####
 
+class GeoIPException(Exception): pass
+
 class GeoIP(object):
     # The flags for GeoIP memory caching.
     # GEOIP_STANDARD - read database from filesystem, uses least memory.

geoip/geoip.py

@@ -1,15 +1,47 @@
+"""
+    This Source Code Form is subject to the terms of the Mozilla Public
+    License, v. 2.0. If a copy of the MPL was not distributed with this
+    file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+    Stand Alone GeoIP lookup service.
+
+    This code is based off of the django.geoip lookup module:
+    https://docs.djangoproject.com/en/dev/ref/contrib/gis/geoip/#module-django.contrib.gis.geoip
+
+    Since this service will be installed to a distributed, production server
+    and since updates may introduce corruption to the dataset that may
+    cause the C API to segfault and kill the python app, this code creates
+    a separate service that isolates such faults.
+
+    By default, this application runs at localhost on port 5309
+
+    send:
+    GET <addr>\n
+
+    receive:
+    On success
+    {'success': {<GeoIP information as JSON>}}\n
+
+    On error:
+    {'error': <Error description string>}\n
+
+"""
 import json
 import logging
 import re
-from base import GeoIP
+from base import GeoIPException, GeoIP
 from gevent.pool import Pool
 from gevent.server import StreamServer
+from statsd import statsd
+
 
-def dotToInt( dottedIp ):
+def dotToInt(dottedIp):
     st = ''
     for octet in dottedIp.split('.'):
         st += "%02x" % int(octet)
-    return int(st,16)
+    return int(st, 16)
+
 
 class GeoServer(StreamServer):
 
@@ -20,9 +52,12 @@ def __init__(self, listener, config, **kw):
         if listener is not None:
             super(GeoServer, self).__init__(listener, **kw)
         self.config = config
-        self.geoip = GeoIP(path=config.GEOIP_PATH,
+        try:
+            self.geoip = GeoIP(path=config.GEOIP_PATH,
                 cache=GeoIP.GEOIP_MEMORY_CACHE)
-        self.log.info('GEOIP starting on port %s...' % config.PORT);
+            self.log.info('GEOIP starting on port %s...' % config.PORT)
+        except GeoIPException, e:
+            self.log.error("Could not start GeoIP server: %s", str(e))
 
     def _error(self, errstr):
         self.log.error("GEOIP: %s" % errstr)
@@ -33,43 +68,75 @@ def _return(self, label='reply', obj={}):
         self.log.info("Returning: %s" % reply)
         return "%s\n" % reply
 
+    def checkConnection(self, socket):
+        try:
+            if self.geoip.country_code('mozilla.org') == 'US':
+                return "200 OK\n"
+        except Exception, e:
+            self.log.error("Failed Health Check: %s", str(e))
+            return "500 Error\n"
+
     def handle(self, socket, address):
+        if statsd is not None:
+            timer = statsd.Timer('GeoIP')
+            req_counter = statsd.counter('GeoIP.request')
+            success_counter = statsd.counter('GeoIP.success')
+            fail_counter = statsd.counter('GeoIP.failure')
+            timer.start()
+        else:
+            timer = None
+            req_counter = 0
+            success_counter = 0
+            fail_counter = 0
         try:
             sock = socket.makefile()
             line = sock.readline()
+            req_counter += 1
             if ' ' not in line:
-                sock.write(self._error('Invalid request\nGET addr'))
+                return sock.write(self._error('Invalid request\nGET addr'))
+            items = line.split(' ')
+            cmd = items[0]
+            addr = items[1].strip().replace('/', '')
+            if cmd.upper() != 'GET':
+                return sock.write(self._error('Invalid Command\nuse GET '))
+            if addr is None:
+                return sock.write(self._error('Missing address'))
+            if 'monitor' in addr:
+                sock.write(self.checkConnection(sock))
+                return sock.flush()
+            reply = self.geoip.city(addr)
+            if reply is None:
+                fail_counter += 1
+                return sock.write(self._error('No information for site'))
             else:
-                cmd, addr = line.split(' ',2)
-                if cmd.upper() != 'GET':
-                    sock.write(self._error('Invalid Command\nuse GET '))
-                elif addr is None:
-                    sock.write(self._error('Missing address'))
-                else:
-                    addr = addr.strip()
-                    reply = self.geoip.city(addr)
-                    if reply is None:
-                        sock.write(self._error('No information for site'))
-                    else:
-                        sock.write(self._return('success',reply))
+                success_counter += 1
+                return sock.write(self._return('success', reply))
         except Exception, e:
-            sock.write(self._error("Unknown error: %s" % e))
+            return sock.write(self._error("Unknown error: %s %s" % (e, line)))
         finally:
+            if timer:
+                timer.stop('Request')
             sock.flush()
 
 if __name__ == "__main__":
     try:
         import settings
-    except ImportError:
-        import pdb; pdb.set_trace()
-        print "uh-oh"
+        """
+        Statsd is HEAVILY django dependent. The following
+        work around attempts to address that.
+        """
+        import os
+        for i in ['STATSD_HOST', 'STATSD_PORT', 'STATSD_PREFIX']:
+            if hasattr(settings, i):
+                print 'loading %s = %s' % (i, getattr(settings, i))
+                os.environ[i] = str(getattr(settings, i))
+    except ImportError, e:
+        print "Cannot run. Terminating."
+        print str(e)
+        exit(-1)
 
     logging.basicConfig(stream=settings.LOG_STREAM, level=settings.LOG_LEVEL)
     pool = Pool(settings.MAX_CONNECTS)
     server = GeoServer((settings.HOST, settings.PORT),
                 config=settings, spawn=pool)
     server.serve_forever()
-#TODO:
-# Add Logging
-# Test.
-

geoip/libgeoip.py

@@ -3,8 +3,6 @@
 from ctypes.util import find_library
 import settings
 
-class GeoIPException(Exception): pass
-
 # Creating the settings dictionary with any settings, if needed.
 GEOIP_SETTINGS = dict((key, getattr(settings, key))
                       for key in ('GEOIP_PATH', 'GEOIP_LIBRARY_PATH', 'GEOIP_COUNTRY', 'GEOIP_CITY')

geoip/settings.py

@@ -11,3 +11,6 @@
 #GEOIP_PATH='./local/GeoLiteCity.dat'
 GEOIP_PATH = 'data/'
 
+STATSD_HOST = 'localhost'
+STATSD_PORT = 8125
+STATSD_PREFIX = 'geoip'

geoip/tests/test_geoip.py

@@ -4,11 +4,13 @@
 from StringIO import StringIO
 from geoip.geoip import GeoServer
 
+
 class AsObj:
 
     def __init__(self, **items):
         self.__dict__.update(items)
 
+
 class FakeSocket(StringIO):
 
     def makefile(self):
@@ -18,24 +20,25 @@ def makefile(self):
 class TestGeoServer(unittest.TestCase):
 
     def setUp(self):
-        self.server = GeoServer(None, AsObj(**{'GEOIP_PATH': 'data/',
-            'PORT':0}))
+        self.server = GeoServer(None, AsObj(**{
+            'GEOIP_PATH': 'data/',
+            'PORT': 0,
+            }))
 
     def test_valid(self):
         fake = FakeSocket('get 8.8.8.8\n')
         self.server.handle(fake, None)
         value = fake.getvalue().split('\n')[1]
         resp = json.loads(value)
         self.failUnless('success' in resp)
-        self.failUnlessEqual(resp['success']['country_code'],'US')
+        self.failUnlessEqual(resp['success']['country_code'], 'US')
 
     def test_bad_command(self):
         fake = FakeSocket('banana\n')
         self.server.handle(fake, None)
         value = fake.getvalue().split('\n')[1]
         self.failUnless('error' in value)
 
-
     def test_bad_request(self):
         fake = FakeSocket('\n')
         self.server.handle(fake, None)

prod-reqs.txt

@@ -4,3 +4,4 @@ argparse==1.2.1
 gevent==0.13.7
 greenlet==0.4.0
 nose==1.1.2
+statsd==0.5.1

setup.py

@@ -1,3 +1,18 @@
-"""
-    Placeholder
-"""
+import os
+
+from setuptools import setup, find_packages
+
+setup(name='geoip_server',
+        version='0.1',
+        description='Stand alone GeoIP server',
+        author='jrconlin',
+        author_email='',
+        license='MPL 2.0',
+        url='https://github.com/jrconlin/geoip_server',
+        include_package_data=True,
+        classifiers=[],
+        packages=find_packages(exclude=['tests']),
+        install_requires=[])
+
+
+