Permalink
Browse files

Refactored the lib to retrieve cities and zipcodes separately, added …

…zipcode import and data
  • Loading branch information...
1 parent 6e133ef commit 26897a014b72e741db5fa743b5d1f1dc0d1c34ad dvir volk committed Mar 27, 2011
Showing with 1,277 additions and 196 deletions.
  1. +3 −0 .gitignore
  2. +4 −4 README.md
  3. +1 −1 src/__init__.py
  4. +51 −0 src/city.py
  5. +23 −5 src/geodis.py
  6. +56 −15 src/iprange.py
  7. +56 −38 src/location.py
  8. +14 −11 src/provider/geonames.py
  9. +38 −0 src/provider/importer.py
  10. +12 −13 src/provider/ip2location.py
  11. +87 −0 src/provider/zipcodes.py
  12. +5 −0 src/us_states.py
  13. +46 −0 src/zipcode.py
  14. +102 −100 test/data/ip2location.csv
  15. +741 −0 test/data/zipcodes.csv
  16. +38 −9 test/test.py
View
@@ -0,0 +1,3 @@
+data/*.csv
+*.pyc
+*.pyo
View
@@ -4,7 +4,7 @@ Geodis - a Redis based geo resolving library
Geodis is a simple python module that allows you to import locations
(currently only cities) and geographical IP ranges into Redis, a fast in-memory NoSQL database.
-It is able to resolve either lat,lon coordinates into city, region and country (based on the closest match),
+It is able to resolve either lat,lon coordinates into zipcode (in the us), city, region and country (based on the closest match),
and/or resolve IP addresses into the same location objects.
Geodis is fast - a single thread, signle process python program can resolve about 1500 locations per second on
@@ -17,12 +17,12 @@ USAGE
>>> import geodis
>>> conn = redis.Redis()
- #getting a location by lat,lon
- >>> print geodis.Location.getByLatLon(31.78,35.21, conn)
+ #getting a city by lat,lon
+ >>> print geodis.City.getByLatLon(31.78,35.21, conn)
Location: {'name': 'West Jerusalem', 'country': 'Israel', 'lon': '35.21961', 'zipcode': '', 'state': 'Jerusalem District', 'lat': '31.78199'}
#getting a location by ip
- >>> print geodis.IPRange.getLocation('62.219.0.221', conn)
+ >>> print geodis.IPRange.getCity('62.219.0.221', conn)
Location: {'name': 'West Jerusalem', 'country': 'Israel', 'lon': '35.21961', 'zipcode': '', 'state': 'Jerusalem District', 'lat': '31.78199'}
View
@@ -25,4 +25,4 @@
#or implied, of Do@.
from iprange import IPRange
-from location import Location
+from city import City
View
@@ -0,0 +1,51 @@
+#Copyright 2011 Do@. All rights reserved.
+#
+#Redistribution and use in source and binary forms, with or without modification, are
+#permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice, this list of
+# conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice, this list
+# of conditions and the following disclaimer in the documentation and/or other materials
+# provided with the distribution.
+#
+#THIS SOFTWARE IS PROVIDED BY Do@ ``AS IS'' AND ANY EXPRESS OR IMPLIED
+#WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+#FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL <COPYRIGHT HOLDER> OR
+#CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+#CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+#SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+#ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+#NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+#ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+#The views and conclusions contained in the software and documentation are those of the
+#authors and should not be interpreted as representing official policies, either expressed
+#or implied, of Do@.
+
+from countries import countries
+import geohash
+
+from location import Location
+
+class City(Location):
+ """
+ Wrapper for a city location object
+ """
+
+ #what we want to save for a city
+ __spec__ = Location.__spec__ + ['country', 'state']
+
+ #key is identical to what we want to save
+ __keyspec__ = None
+
+ def __init__(self, **kwargs):
+
+ super(City, self).__init__(**kwargs)
+
+ self.country = countries.get( kwargs.get('country', None), kwargs.get('country', '')).strip()
+ self.state = kwargs.get('state', '').strip()
+
+
+
View
@@ -33,19 +33,21 @@
from provider.geonames import GeonamesImporter
from provider.ip2location import IP2LocationImporter
+from provider.zipcodes import ZIPImporter
from iprange import IPRange
-from location import Location
+from city import City
__author__="dvirsky"
__date__ ="$Mar 25, 2011 4:44:22 PM$"
redis_host = 'localhost'
redis_port = 6379
+redis_db = 0
def importGeonames(fileName):
global redis_host, redis_port
- importer = GeonamesImporter(fileName, redis_host, redis_port)
+ importer = GeonamesImporter(fileName, redis_host, redis_port, redis_db)
if not importer.runImport():
print "Could not import geonames database..."
sys.exit(1)
@@ -56,7 +58,16 @@ def importGeonames(fileName):
def importIP2Location(fileName):
global redis_host, redis_port
- importer = IP2LocationImporter(fileName, redis_host, redis_port)
+ importer = IP2LocationImporter(fileName, redis_host, redis_port, redis_db)
+ if not importer.runImport():
+ print "Could not import geonames database..."
+ sys.exit(1)
+
+
+def importZIPCode(fileName):
+
+ global redis_host, redis_port
+ importer = ZIPImporter(fileName, redis_host, redis_port, redis_db)
if not importer.runImport():
print "Could not import geonames database..."
sys.exit(1)
@@ -65,13 +76,13 @@ def importIP2Location(fileName):
def resolveIP(ip):
r = redis.Redis(host = redis_host, port = redis_port)
- loc = IPRange.getLocation(ip, r)
+ loc = IPRange.getCity(ip, r)
print loc
def resolveCoords(lat, lon):
r = redis.Redis(host = redis_host, port = redis_port)
- loc = Location.getByLatLon(lat, lon, r)
+ loc = City.getByLatLon(lat, lon, r)
print loc
@@ -91,13 +102,17 @@ def resolveCoords(lat, lon):
parser.add_option("-i", "--import_ip2coutnry", dest="import_ip2location",
action='store_true', default=False,
help='Import ip ranges from ip2country.com dumps')
+ parser.add_option("-z", "--import_zipcodes", dest="import_zipcodes",
+ action='store_true', default=False,
+ help='Import zipcodes')
parser.add_option("-f", "--file", dest="import_file",
help="Location of the file we want to import", metavar="FILE")
parser.add_option("-p", "--resolve_ip", dest="resolve_ip", default = None,
help="resolve an ip address to location", metavar="IP_ADDR")
+
parser.add_option("-l", "--resolve_latlon", dest="resolve_latlon", default = None,
help="resolve an lat,lon pair into location", metavar="LAT,LON")
@@ -108,6 +123,9 @@ def resolveCoords(lat, lon):
elif options.import_ip2location:
importIP2Location(options.import_file)
+
+ elif options.import_zipcodes:
+ importZIPCode(options.import_file)
elif options.resolve_ip:
resolveIP(options.resolve_ip)
View
@@ -23,23 +23,29 @@
#The views and conclusions contained in the software and documentation are those of the
#authors and should not be interpreted as representing official policies, either expressed
#or implied, of Do@.
-import socket, struct
-from location import Location
+
+
+import socket, struct, re
+from city import City
+from zipcode import ZIPCode
import geohash
import struct
class IPRange(object):
_indexKey = 'iprange:locations'
- def __init__(self, rangeMin, rangeMax, lat, lon):
+ def __init__(self, rangeMin, rangeMax, lat, lon, zipcode = ''):
self.rangeMin = rangeMin
self.rangeMax = rangeMax
+ self.lat = lat
+ self.lon = lon
+ self.zipcode = zipcode
#encode a numeric geohash key
- self.geoKey = geohash.encode_uint64(lat, lon)
+ self.geoKey = geohash.encode(lat, lon)
- self.key = '%s:%s' % (self.rangeMin, self.rangeMax)
+ self.key = '%s:%s:%s' % (self.rangeMin, self.rangeMax, self.zipcode)
def save(self, redisConn):
"""
@@ -55,16 +61,13 @@ def __str__(self):
textual representation
"""
return "IPRange: %s" % self.__dict__
-
+
@staticmethod
- def getLocation(ip, redisConn):
+ def get(ip, redisConn):
"""
- Get location object by resolving an IP address
- @param ip IPv4 address string (e.g. 127.0.0.1)
- @oaram redisConn redis connection to the database
- @return a Location object if we can resolve this ip, else None
+ Get a range and all its data by ip
"""
-
+
ipnum = IPRange.ip2long(ip)
#get the location record from redis
@@ -76,18 +79,56 @@ def getLocation(ip, redisConn):
#extract location id
try:
geoKey,rng = record[0][0].split('@')
- geoKey = int(geoKey)
- rngMin, rngMax = (int(x) for x in rng.split(':'))
+
+ lat,lon = geohash.decode(geoKey)
+
+ rngMin, rngMax, zipcode = rng.split(':')
+ rngMin = int(rngMin)
+ rngMax = int(rngMax)
except IndexError:
return None
#address not in any range
if not rngMin <= ipnum <= rngMax:
return None
+ return IPRange(rngMin, rngMax, lat, lon, zipcode)
+
+ @staticmethod
+ def getZIP(ip, redisConn):
+ """
+ Get a zipcode location object based on an IP
+ will return None if you are outside the US
+ """
+
+ range = IPRange.get(ip, redisConn)
+ if not range or not re.match('^[0-9]{5}$', range.zipcode):
+ return None
+
+ return ZIPCode.load('ZIPCode:%s' % range.zipcode, redisConn)
+
+
+
+
+
+
+ @staticmethod
+ def getCity(ip, redisConn):
+ """
+ Get location object by resolving an IP address
+ @param ip IPv4 address string (e.g. 127.0.0.1)
+ @oaram redisConn redis connection to the database
+ @return a Location object if we can resolve this ip, else None
+ """
+
+ range = IPRange.get(ip, redisConn)
+ if not range:
+ return None
+
+
#load a location by the
- return Location.getByGeohash(geoKey, redisConn)
+ return City.getByGeohash(geohash.encode_uint64(range.lat, range.lon), redisConn)
@staticmethod
Oops, something went wrong.

0 comments on commit 26897a0

Please sign in to comment.