Permalink
Browse files

Added an import command for states/provinces, and added a fix for Que…

…bec in the geocoder
  • Loading branch information...
bkonkle
bkonkle committed Jan 17, 2011
1 parent b3e395b commit 4fc8ad034ec345976b220875942ba850191f26b9
View
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
import re
from geonames.models import Geoname, GeonameAlternateName, Country
@@ -7,7 +8,7 @@
us_address_re = re.compile(r'(?:(?P<number>\d+)\s+(?P<street>[\w\s]+),?\s+)?(?P<city>[\w\s]+),?\s+(?P<state>%s|%s)' % (us_states, us_state_abbrs), re.I)
can_prov_abbrs = 'AB|BC|MB|NB|NL|NT|NS|NU|ON|PE|QC|SK|YT'
-can_provinces = 'Alberta|British Columbia|Manitoba|New Brunswick|Newfoundland and Labrador|Northwest Territories|Nova Scotia|Nunavut|Ontario|Prince Edward Island|Quebec|Saskatchewan|Yukon'
+can_provinces = 'Alberta|British Columbia|Manitoba|New Brunswick|Newfoundland and Labrador|Northwest Territories|Nova Scotia|Nunavut|Ontario|Prince Edward Island|Quebec|Québec|Saskatchewan|Yukon'
can_city_prov_re = re.compile(r'(?P<city>[\w\s]+),?\s+(?P<province>%s|%s)' % (can_provinces, can_prov_abbrs), re.I)
city_country_re = re.compile(r'(?P<city>[\w\s]+?),?\s+(?P<country>[\w\s]+)', re.I)
@@ -19,6 +20,10 @@ def geocode(query, first=True):
then look for a Geoname object to match it. By default, it returns the
first result. If first is False, however, it returns a list of all results.
"""
+ # Quick fix for Québec
+ query = query.replace('Quebec', u'Québec')
+ query = query.replace('quebec', u'Québec')
+
# If the query is two or three letters, try the appropriate ISO code first
try:
if len(query) == 2:
@@ -0,0 +1,65 @@
+import sys
+
+from django.core.management.base import NoArgsCommand
+from django.db import connections
+
+from geonames.models import Country, Geoname
+
+"""
+Generates geonames for all first-order administrative divisions, such as US
+states and Canadia provinces. To run this command, first download the
+allCountries.zip file::
+
+ wget http://download.geonames.org/export/dump/allCountries.zip
+
+After unzipping the allCountries.zip file and placing both files in the project
+directory, you can run the command.
+"""
+
+class Command(NoArgsCommand):
+ help = "Generates the countries geonames objects in JSON format"
+
+ def handle_noargs(self, **options):
+ print "Importing Geoname objects for the AMD1 geonmes in countryInfo.txt"
+ cursor = connections['default'].cursor()
+ cursor.execute('BEGIN')
+ with open('allCountries.txt') as fd:
+ i = 0
+ for line in fd:
+ i += 1
+ if i % 100000 == 0:
+ sys.stdout.write('.')
+ sys.stdout.flush()
+
+ fields = line.split('\t')
+ geoname_id, name, ascii_name = fields[:3]
+ latitude, longitude, fclass, fcode, country_id, cc2 = fields[4:10]
+
+ if not fcode == 'ADM1':
+ continue
+
+ population, elevation, gtopo30 = fields[14:17]
+ moddate = fields[18]
+ if elevation == '':
+ elevation = 0
+
+ timezone_id = None
+ name = unicode(name,'utf-8')
+ admin1 = fields[10]
+ admin2 = fields[11]
+ admin3 = fields[12]
+ admin4 = fields[13]
+ admin1_id, admin2_id, admin3_id, admin4_id = [None] * 4
+
+ try:
+ # Delete existing entries first, then insert
+ cursor.execute(u"DELETE FROM geoname WHERE id = %s", (geoname_id,))
+ cursor.execute(u"INSERT INTO geoname (id, name, ascii_name, point, fclass, fcode, country_id, cc2, admin1_id, admin2_id, admin3_id, admin4_id, population, elevation, gtopo30, timezone_id, moddate) VALUES (%s, %s, %s, GeomFromText(%s, 4326), %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)", (geoname_id, name, ascii_name, 'POINT(%s %s)' % (longitude, latitude), fclass, fcode, country_id, cc2, admin1_id, admin2_id, admin3_id, admin4_id, population, elevation, gtopo30, timezone_id, moddate))
+ except Exception, e:
+ print 'Error: %s' % e
+ continue
+
+ print 'Done!'
+ cursor.execute('COMMIT')
+
+ print "Complete! States and Provinces should now be available for querying."
@@ -67,6 +67,9 @@ def __init__(self, host=None, user=None, password=None, db=None,
FILES.append(PREFIX + 'cities%s.zip' % self.cities)
self.zip_files.append('cities%s.zip' % self.cities)
self.geonames_file = 'cities%s.txt' % self.cities
+
+ import ipdb
+ ipdb.set_trace()
def pre_import(self):
pass

0 comments on commit 4fc8ad0

Please sign in to comment.