Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP

Loading…

Bug 793395 - Convert socket lookup to native #436

Closed
wants to merge 1 commit into from

2 participants

@jrconlin

This uses the native geoip lookup functions. If there's a problem
with this in the future, the patch code is available.
(Update: removed dat files. Please pull files directly from maxmind.com)

jrconlin Bug 793395 - Convert socket lookup to native
This uses the native geoip lookup functions. If there's a problem
with this in the future, the patch code is available.
(Update: removed dat files. Please pull files directly from maxmind.com)
6895412
@cvan
Owner

is this good to be merged in?

@cvan
Owner

is this out of date?

@cvan
Owner

I'm assuming this is stale.

@cvan cvan closed this
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Commits on Sep 27, 2012
  1. Bug 793395 - Convert socket lookup to native

    jrconlin authored
    This uses the native geoip lookup functions. If there's a problem
    with this in the future, the patch code is available.
    (Update: removed dat files. Please pull files directly from maxmind.com)
This page is out of date. Refresh to see the latest.
View
77 lib/geoip/__init__.py
@@ -1,77 +0,0 @@
-import json
-import logging
-import socket
-from statsd import statsd
-
-from mkt import regions
-
-
-class GeoIP:
- """ Call to local GeoIP server to resolve an IP to Geo Info block """
-
- def __init__(self, settings):
- # I attempt to make this fairly fault proof. We don't want to
- # call the service directly, because bad data would cause the
- # C library to segfault and take out the server, so we isolate
- # that data to an external service. We also pack in as many
- # default values as possible to prevent unexpected results.
- self.noop = getattr(settings, 'GEOIP_NOOP', False)
- self.timeout = float(getattr(settings, 'GEOIP_DEFAULT_TIMEOUT', .2))
- self.host = getattr(settings, 'GEOIP_HOST', 'localhost')
- self.port = int(getattr(settings, 'GEOIP_PORT', '5309'))
- self.default_val = getattr(settings, 'GEOIP_DEFAULT_VAL',
- regions.US.slug).lower()
- # Changing this value is ONLY useful for testing.
- self.socket_lib = getattr(settings, 'GEOIP_TEST_SOCKETLIB',
- socket)
-
- def lookup(self, address):
- """ Resolve an IP address to a block of geo information.
-
- If a given address is unresolvable, return the default
- as defined by the settings, or "worldwide".
- """
- if self.noop:
- return self.default_val
- sock = self.socket_lib
- gsocket = sock.socket(socket.AF_INET, socket.SOCK_STREAM)
- gsocket.settimeout(self.timeout)
- gsocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
- with statsd.timer('z.geoip'):
- try:
- gsocket.connect((self.host, self.port))
- # Remember, we're using a timeout, so don't call makefile()!
- send = 'GET %s\n' % address
- tsent = 0
- while tsent < len(send):
- sent = gsocket.send(send[tsent:])
- if sent == 0:
- raise IOError('Socket connection broken')
- tsent += sent
- rcv = ''
- while True:
- try:
- rcvm = gsocket.recv(1)
- if len(rcvm) == 0:
- break
- except StopIteration:
- # This is required for unit testing.
- break
- rcv += rcvm
- reply = json.loads(rcv)
- if 'error' in reply:
- return self.default_val
- else:
- return reply['success']['country_code'].lower()
- except socket.timeout:
- logging.warn('GeoIP server timeout. '
- 'Returning default')
- return self.default_val
- except IOError, e:
- logging.error('GeoIP server down or missing')
- return self.default_val
- except Exception, e:
- logging.error('Unknown exception: %s', str(e))
- return self.default_val
- finally:
- gsocket.close()
View
44 lib/geoip/tests/test_geoip.py
@@ -1,44 +0,0 @@
-import socket
-import mock
-from nose.tools import eq_
-
-import amo.tests
-
-from lib.geoip import GeoIP
-
-
-class NOOP_Settings:
-
- GEOIP_DEFAULT_VAL = 'worldwide'
-
-
-class GeoIPTest(amo.tests.TestCase):
-
- def setUp(self):
- # Use GEOIP_NOOP to always return the default value.
- # This *should* be properly tested against a call to a GeoIP server.
- self.geoip = GeoIP(NOOP_Settings)
-
- @mock.patch('socket.socket')
- def test_lookup(self, mock_socket):
- send_value = 'mozilla.com'
- mock_socket.return_value.connect.return_value = True
- rcv = list('{"success":{"country_code":"us"}}')
- # 5 = prefixed "GET " and new line
- mock_socket.return_value.send.return_value = len(send_value) + 5
- mock_socket.return_value.recv.side_effect = rcv
- result = self.geoip.lookup(send_value)
- eq_(result, 'us')
-
- @mock.patch('socket.socket')
- def test_no_connect(self, mock_socket):
- mock_socket.return_value.connect.side_effect = IOError
- result = self.geoip.lookup('mozilla.com')
- eq_(result, 'worldwide')
-
- @mock.patch('socket.socket')
- def test_timeout(self, mock_socket):
- mock_socket.return_value.connect.return_value = True
- mock_socket.return_value.send.side_effect = socket.timeout
- result = self.geoip.lookup('mozilla.com')
- eq_(result, 'worldwide')
View
5 mkt/settings.py
@@ -345,10 +345,9 @@ def APPCACHE_MEDIA_DEBUG():
# This flag overrides the GeoIP server functions and will force the
# return of the GEOIP_DEFAULT_VAL
GEOIP_NOOP = 1
-GEOIP_HOST = 'localhost'
-GEOIP_PORT = '5309'
+import pdb; pdb.set_trace()
+GEOIP_PATH = os.path.join(MEDIA_ROOT, 'geoip')
GEOIP_DEFAULT_VAL = 'us'
-GEOIP_DEFAULT_TIMEOUT = .2
# A smaller range of languages for the Marketplace.
AMO_LANGUAGES = ('en-US', 'es', 'pt-BR')
View
16 mkt/site/middleware.py
@@ -4,6 +4,7 @@
from django.conf import settings
from django.http import SimpleCookie, HttpRequest
from django.utils.cache import patch_vary_headers
+from django.contrib.gis.geoip import GeoIP
import tower
@@ -12,7 +13,6 @@
from amo.utils import urlparams
import mkt
-from lib.geoip import GeoIP
def _set_cookie(self, key, value='', max_age=None, expires=None, path='/',
@@ -169,7 +169,7 @@ class RegionMiddleware(object):
"""Figure out the user's region and store it in a cookie."""
def __init__(self):
- self.geoip = GeoIP(settings)
+ self.geoip = GeoIP()
def process_request(self, request):
regions = mkt.regions.REGIONS_DICT
@@ -198,7 +198,17 @@ def process_request(self, request):
# valid region, try from the IP
if (request.LANG.lower() not in
request.META.get('HTTP_ACCEPT_LANGUAGE', '').lower()):
- ip_reg = self.geoip.lookup(request.META.get('REMOTE_ADDR'))
+ if not (getattr(settings, 'GEOIP_NOOP', True)):
+ try:
+ country = self.geoip.country(
+ request.meta.get('HTTP_REMOTE_ADDR'))
+ ip_reg = country['country_code'].lower()
+
+ except Exception, e:
+ #logger.error('Locator returned exception %s' % str(e))
+ ip_reg = getattr(settings, 'GEOIP_DEFAULT_VAL', 'us')
+ else:
+ ip_reg = getattr(settings, 'GEOIP_DEFAULT_VAL', 'us')
for name, region in choices:
if ip_reg == name:
reg = region.slug
View
19 mkt/site/tests/test_middleware.py
@@ -1,5 +1,3 @@
-import socket
-
from django.conf import settings
import mock
@@ -360,23 +358,6 @@ def test_geoip_missing_lang(self):
r = self.client.get('/', REMOTE_ADDR='mozilla.com')
eq_(r.cookies['region'].value, 'us')
- @mock.patch.object(settings, 'GEOIP_DEFAULT_VAL', 'us')
- @mock.patch('socket.socket')
- def test_geoip_down(self, mock_socket):
- """ Test that we fail gracefully if the GeoIP server is down. """
- mock_socket.connect.side_effect = IOError
- r = self.client.get('/', REMOTE_ADDR='mozilla.com')
- eq_(r.cookies['region'].value, 'us')
-
- @mock.patch.object(settings, 'GEOIP_DEFAULT_VAL', 'us')
- @mock.patch('socket.socket')
- def test_geoip_timeout(self, mock_socket):
- """ Test that we fail gracefully if the GeoIP server times out. """
- mock_socket.return_value.connect.return_value = True
- mock_socket.return_value.send.side_effect = socket.timeout
- r = self.client.get('/', REMOTE_ADDR='mozilla.com')
- eq_(r.cookies['region'].value, 'us')
-
class TestVaryMiddleware(MiddlewareCase):
Something went wrong with that request. Please try again.