Skip to content
Browse files

Moved everything into subfolder and added a setup script

  • Loading branch information...
1 parent 61f3c1a commit d02f7d5f2322c55aa3d2234dbdc2dc613eb12cf1 @maketolearn maketolearn committed May 23, 2011
View
BIN .DS_Store
Binary file not shown.
View
0 build/lib/geonames/__init__.py
No changes.
View
68 build/lib/geonames/decorators.py
@@ -0,0 +1,68 @@
+# -*- coding: utf-8 -*-
+
+# The contents of this file are subject to the Common Public Attribution License
+# Version 1.0 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.cpal-dev.com/cpaldemoopen/license. The License is based on the
+# Mozilla Public License Version 1.1 but Sections 14 and 15 have been added to cover
+# use of software over a computer network and provide for limited attribution for
+# the Original Developer. In addition, Exhibit A has been modified to be consistent
+# with Exhibit B.
+
+# Software distributed under the License is distributed on an "AS IS" basis, WITHOUT
+# WARRANTY OF ANY KIND, either express or implied. See the License for the specific
+# anguage governing rights and limitations under the License.
+
+# The Original Code is the byNotes project
+
+# The Initial Developer of the Original Code is Alberto García Hierro
+# The Original Developer is Alberto García Hierro
+
+# All portions of the code written by Alberto García Hierro are
+# Copyright (C) 2008 Alberto García Hierro
+# All Rights Reserved.
+
+from django.core.cache import cache
+
+def cache_set(key, value):
+ cache.set(key, value)
+ return value
+
+def _cached(func):
+ def cached_func(self):
+ key = 'cached_property_%s_%s_%s' % \
+ (self.__class__.__name__, func.__name__, self.pk)
+ val = cache.get(key)
+ return cache_set(key, func(self)) if val is None else val
+
+ cached_func.__name__ = func.__name__
+ return cached_func
+
+def cached_property(func):
+ return property(_cached(func))
+
+def _stored(func):
+ key = '_cached_%s' % func.__name__
+ def stored_func(self):
+ if not hasattr(self, key):
+ setattr(self, key, func(self))
+ return getattr(self, key)
+
+ stored_func.__name__ = func.__name__
+ return stored_func
+
+def stored_property(func):
+ return property(_stored(func))
+
+def full_cached_property(func):
+ return stored_property(_cached(func))
+
+def cached_method(func):
+ def cached_func(self, *args, **kwargs):
+ key = 'cached_method_%s_%s_%s_%s_%s' % \
+ (self.__class__.__name__, func.__name__, self.pk, hash(args),
+ hash(kwargs.iteritems()))
+ val = cache.get(key)
+ return cache_set(key, func(self, *args, **kwargs)) if val is None else val
+
+ return cached_func
View
33 build/lib/geonames/fcodes.py
@@ -0,0 +1,33 @@
+# This file is part of Django-Geonames
+# Copyright (c) 2008, Alberto Garcia Hierro
+# See LICENSE file for details
+
+FCODES = [
+ ugettext_noop('first-order administrative division'),
+ ugettext_noop('second-order administrative division'),
+ ugettext_noop('third-order administrative division'),
+ ugettext_noop('fourth-order administrative division'),
+ ugettext_noop('administrative division'),
+ ugettext_noop('leased area'),
+ ugettext_noop('political entity'),
+ ugettext_noop('dependent political entity'),
+ ugettext_noop('freely associated state'),
+ ugettext_noop('independent political entity'),
+ ugettext_noop('section of independent political entity'),
+ ugettext_noop('semi-independent political entity'),
+ ugettext_noop('parish'),
+ ugettext_noop('territory'),
+ ugettext_noop('zone'),
+ ugettext_noop('buffer zone'),
+ ugettext_noop('populated place'),
+ ugettext_noop('seat of a first-order administrative division'),
+ ugettext_noop('capital of a political entity'),
+ ugettext_noop('seat of government of a political entity'),
+ ugettext_noop('populated locality'),
+ ugettext_noop('abandoned populated place'),
+ ugettext_noop('religious populated place'),
+ ugettext_noop('populated places'),
+ ugettext_noop('destroyed populated place'),
+ ugettext_noop('section of populated place'),
+ ugettext_noop('israeli settlement'),
+]
View
0 build/lib/geonames/management/__init__.py
No changes.
View
0 build/lib/geonames/management/commands/__init__.py
No changes.
View
733 build/lib/geonames/management/commands/geonames_import.py
@@ -0,0 +1,733 @@
+import os
+import sys
+from optparse import OptionParser
+from warnings import filterwarnings
+from getpass import getpass
+from datetime import date
+from django.conf import settings
+from django.core.management.base import BaseCommand
+from django.db import connections, DEFAULT_DB_ALIAS
+
+FILES = [
+ 'http://download.geonames.org/export/dump/allCountries.zip',
+ 'http://download.geonames.org/export/dump/alternateNames.zip',
+ 'http://download.geonames.org/export/dump/admin1CodesASCII.txt',
+ 'http://download.geonames.org/export/dump/admin2Codes.txt',
+ 'http://download.geonames.org/export/dump/featureCodes_en.txt',
+ 'http://download.geonames.org/export/dump/timeZones.txt',
+ 'http://download.geonames.org/export/dump/countryInfo.txt',
+]
+
+CONTINENT_CODES = [
+ ('AF', 'Africa' , 6255146),
+ ('AS', 'Asia', 6255147),
+ ('EU', 'Europe', 6255148),
+ ('NA', 'North America', 6255149),
+ ('OC', 'Oceania', 6255151),
+ ('SA', 'South America', 6255150),
+ ('AN', 'Antarctica', 6255152),
+]
+
+class GeonamesImporter(object):
+ def __init__(self, host=None, user=None, password=None, db=None, tmpdir='tmp'):
+ self.user = user
+ self.password = password
+ self.db = db
+ self.host = host
+ self.conn = None
+ self.tmpdir = tmpdir
+ self.curdir = os.getcwd()
+ self.time_zones = {}
+ self.admin1_codes = {}
+ self.admin2_codes = {}
+ self.admin3_codes = {}
+ self.admin4_codes = {}
+
+ def pre_import(self):
+ pass
+
+ def post_import(self):
+ pass
+
+ def begin(self):
+ pass
+
+ def commit(self):
+ pass
+
+ def last_row_id(self, table=None, pk=None):
+ raise NotImplementedError('This is a generic importer, use one of the subclasses')
+
+ def get_db_conn(self):
+ raise NotImplementedError('This is a generic importer, use one of the subclasses')
+
+ def set_import_date(self):
+ raise NotImplementedError('This is a generic importer, use one of the subclasses')
+
+ def fetch(self):
+ try:
+ os.mkdir(self.tmpdir)
+ os.chdir(self.tmpdir)
+ except OSError:
+ os.chdir(self.tmpdir)
+ print 'Temporary directory %s exists, using already downloaded data'%self.tmpdir
+ return
+
+ for f in FILES:
+ if os.system('wget %s' % f) != 0:
+ print 'Error fetching %s' % os.path.basename(f)
+ sys.exit(1)
+
+ for f in ('allCountries.zip', 'alternateNames.zip'):
+ if os.system('unzip %s' % f) != 0:
+ print 'Error unzipping %s' % f
+ sys.exit(1)
+
+ def cleanup(self):
+ os.chdir(self.curdir)
+ for f in os.listdir(self.tmpdir):
+ os.unlink('%s/%s' % (self.tmpdir, f))
+ os.rmdir(self.tmpdir)
+
+ def handle_exception(self, e, line=None):
+ print e
+ sys.exit(1)
+
+ def table_count(self, table):
+ self.cursor.execute(u'SELECT COUNT(*) FROM %s' % table)
+ return self.cursor.fetchone()[0]
+
+ def import_fcodes(self):
+ print 'Importing feature codes'
+ fd = open('featureCodes_en.txt')
+ line = fd.readline()[:-1]
+ while line:
+ codes, name, desc = line.split('\t')
+ try:
+ fclass, code = codes.split('.')
+ except ValueError:
+ line = fd.readline()[:-1]
+ continue
+ try:
+ self.cursor.execute(u'INSERT INTO feature_code (code, fclass, name, description) VALUES (%s, %s, %s, %s)', (code, fclass, name, desc))
+ except Exception, e:
+ if 'Duplicate' in str(e):
+ #print "I think the data was already entered."
+ return True
+ self.handle_exception(e, line)
+
+ line = fd.readline()[:-1]
+ fd.close()
+ print '%d feature codes imported' % self.table_count('feature_code')
+
+ def import_language_codes(self):
+ print 'Importing language codes'
+ fd = open('iso-languagecodes.txt')
+ fd.readline()
+ line = fd.readline()[:-1]
+ while line:
+ fields = line.split('\t')
+ try:
+ self.cursor.execute(u'INSERT INTO iso_language (iso_639_3, iso_639_2, iso_639_1, language_name) VALUES (%s, %s, %s, %s)', fields)
+ except Exception, e:
+ if 'Duplicate' in str(e):
+ #print "I think the data was already entered."
+ return True
+ self.handle_exception(e)
+
+ line = fd.readline()[:-1]
+ fd.close()
+ print '%d language codes imported' % self.table_count('iso_language')
+
+ def import_alternate_names(self):
+ print 'Importing alternate names (this is going to take a while)'
+ if hasattr(self,'import_file'):
+ self.import_file('alternate_name','alternateNames.txt')
+ fd = open('alternateNames.txt')
+ line = fd.readline()[:-1]
+ while line:
+ id, geoname_id, lang, name, preferred, short = line.split('\t')
+ if preferred in ('', '0'):
+ preferred = False
+ else:
+ preferred = True
+ if short in ('', '0'):
+ short = False
+ else:
+ short = True
+ try:
+ self.cursor.execute(u'INSERT INTO alternate_name (id, geoname_id, language, name, preferred, short) VALUES (%s, %s, %s, %s, %s, %s)', (id, geoname_id, lang, name, preferred, short))
+ except Exception, e:
+ if 'Duplicate' in str(e):
+ #print "I think the data was already entered."
+ return True
+ self.handle_exception(e, line)
+ line = fd.readline()[:-1]
+ fd.close()
+ print '%d alternate names imported' % self.table_count('alternate_name')
+
+ def import_time_zones(self):
+ print 'Importing time zones'
+ fd = open('timeZones.txt')
+ fd.readline()
+ line = fd.readline()[:-1]
+ while line:
+ name, gmt, dst = line.split('\t')
+ try:
+ self.cursor.execute(u'INSERT INTO time_zone (name, gmt_offset, dst_offset) VALUES (%s, %s, %s)', (name, gmt, dst))
+ except Exception, e:
+ if 'Duplicate' in str(e):
+ #print "I think the data was already entered."
+ return True
+ self.handle_exception(e, line)
+
+ self.time_zones[name] = self.last_row_id('time_zone', 'id')
+ line = fd.readline()[:-1]
+ fd.close()
+ print '%d time zones imported' % self.table_count('time_zone')
+
+ def import_continent_codes(self):
+ for continent in CONTINENT_CODES:
+ try:
+ self.cursor.execute(u'INSERT INTO continent (code, name, geoname_id) VALUES (%s, %s, %s)', continent)
+ except Exception, e:
+ if 'Duplicate' in str(e):
+ #print "I think the data was already entered."
+ return True
+ self.handle_exception(e)
+ print '%d continent codes imported' % self.table_count('continent')
+
+ def import_countries(self):
+ print 'Importing countries'
+ fd = open('countryInfo.txt')
+ fd.readline()
+ line = fd.readline()[:-1]
+ while line:
+ if line[0] == '#' or line.startswith('ISO') or line.startswith('CS'):
+ line = fd.readline()[:-1]
+ continue
+ fields = line.split('\t')
+ #if len(fields) == 18:
+ # fields.append('')
+ fields[6] = fields[6].replace(',', '')
+ fields[7] = fields[7].replace(',', '')
+ if fields[6] == '':
+ fields[6] = 0
+ try:
+ self.cursor.execute(u'INSERT INTO country (iso_alpha2, iso_alpha3, iso_numeric, fips_code, name, capital, area, population, continent_id, tld, currency_code, currency_name, phone_prefix, postal_code_fmt, postal_code_re, languages, geoname_id) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)', fields[:17])
+ except Exception, e:
+ if 'Duplicate' in str(e):
+ #print "I think the data was already entered."
+ return True
+ self.handle_exception(e, line)
+ line = fd.readline()[:-1]
+ fd.close()
+ print '%d countries imported' % self.table_count('country')
+
+ def import_first_level_adm(self):
+ print 'Importing first level administrative divisions'
+ fd = open('admin1CodesASCII.txt')
+ line = fd.readline()[:-1]
+ while line:
+ country_and_code, name, ascii_name, geoname_id = line.split('\t')
+ country_id, code = country_and_code.split('.')
+ try:
+ name = unicode(name,'utf-8')
+ except Exception, inst:
+ raise Exception("Encountered an error trying to encode the value for this line:\n%s\n\nThe error was: %s" (line, inst))
+ try:
+ print(u'INSERT INTO admin1_code (country_id, geoname_id, code, name, ascii_name) VALUES (%s, %s, %s, %s, %s)' % (country_id, geoname_id, code, name, ascii_name))
+ self.cursor.execute(u'INSERT INTO admin1_code (country_id, geoname_id, code, name, ascii_name) VALUES (%s, %s, %s, %s, %s)', (country_id, geoname_id, code, name, ascii_name))
+ except Exception, e:
+ print(u'INSERT INTO admin1_code (country_id, geoname_id, code, name, ascii_name) VALUES (%s, %s, %s, %s, %s)' % (country_id, geoname_id, code, name, ascii_name))
+ if 'Duplicate' in str(e):
+ #print "I think the data was already entered."
+ return True
+ print "Got an error %s" % (e)
+ self.handle_exception(e, line)
+
+ self.admin1_codes.setdefault(country_id, {})
+ self.admin1_codes[country_id][code] = self.last_row_id('admin1_code', 'id')
+ line = fd.readline()[:-1]
+ fd.close()
+ print '%d first level administrative divisions imported' % self.table_count('admin1_code')
+
+ def import_second_level_adm(self):
+ print 'Importing second level administrative divisions'
+ fd = open('admin2Codes.txt')
+ line = fd.readline()[:-1]
+ while line:
+ codes, name, ascii_name, geoname_id = line.split('\t')
+ country_id, adm1, code = codes.split('.', 2)
+ try:
+ name = unicode(name,'utf-8')
+ except Exception, inst:
+ raise Exception("Encountered an error trying to encode the value for this line:\n%s\n\nThe error was: %s" (line, inst))
+ try:
+ admin1 = self.admin1_codes[country_id][adm1]
+ except KeyError:
+ admin1 = None
+ try:
+ self.cursor.execute(u'INSERT INTO admin2_code (country_id, admin1_id, geoname_id, code, name, ascii_name) VALUES (%s, %s, %s, %s, %s, %s)', (country_id, admin1, geoname_id, code, name, ascii_name))
+ except Exception, e:
+ if 'Duplicate' in str(e):
+ #print "I think the data was already entered."
+ return True
+ self.handle_exception(e, line)
+
+ self.admin2_codes.setdefault(country_id, {})
+ self.admin2_codes[country_id].setdefault(adm1, {})
+ self.admin2_codes[country_id][adm1][code] = self.last_row_id('admin2_code', 'id')
+ line = fd.readline()[:-1]
+ fd.close()
+ print '%d second level administrative divisions imported' % self.table_count('admin2_code')
+
+ def import_third_level_adm(self):
+ print 'Importing third level administrative divisions'
+ fd = open('allCountries.txt')
+ line = fd.readline()[:-1]
+ while line:
+ fields = line.split('\t')
+ fcode = fields[7]
+ if fcode != 'ADM3':
+ line = fd.readline()[:-1]
+ continue
+ geoname_id = fields[0]
+ name = fields[1]
+ try:
+ name = unicode(name,'utf-8')
+ except Exception, inst:
+ raise Exception("Encountered an error trying to encode the value for this line:\n%s\n\nThe error was: %s" (line, inst))
+ ascii_name = fields[2]
+ country_id = fields[8]
+ admin1 = fields[10]
+ admin2 = fields[11]
+ admin3 = fields[12]
+ admin1_id, admin2_id = [None] * 2
+ if admin1:
+ try:
+ admin1_id = self.admin1_codes[country_id][admin1]
+ except KeyError:
+ pass
+ if admin2:
+ try:
+ admin2_id = self.admin2_codes[country_id][admin1][admin2]
+ except KeyError:
+ pass
+ try:
+ self.cursor.execute(u'INSERT INTO admin3_code (country_id, admin1_id, admin2_id, geoname_id, code, name, ascii_name) VALUES (%s, %s, %s, %s, %s, %s, %s)', (country_id, admin1_id, admin2_id, geoname_id, admin3, name, ascii_name))
+ except Exception, e:
+ if 'Duplicate' in str(e):
+ #print "I think the data was already entered."
+ return True
+ self.handle_exception(e, line)
+
+ self.admin3_codes.setdefault(country_id, {})
+ self.admin3_codes[country_id].setdefault(admin1, {})
+ self.admin3_codes[country_id][admin1].setdefault(admin2, {})
+ self.admin3_codes[country_id][admin1][admin2][admin3] = self.last_row_id('admin3_code', 'id')
+
+ line = fd.readline()[:-1]
+
+ fd.close()
+ print '%d third level administrative divisions imported' % self.table_count('admin3_code')
+
+ def import_fourth_level_adm(self):
+ print 'Importing fourth level administrative divisions'
+ fd = open('allCountries.txt')
+ line = fd.readline()[:-1]
+ while line:
+ fields = line.split('\t')
+ fcode = fields[7]
+ if fcode != 'ADM4':
+ line = fd.readline()[:-1]
+ continue
+ geoname_id = fields[0]
+ name = fields[1]
+ try:
+ name = unicode(name,'utf-8')
+ except Exception, inst:
+ raise Exception("Encountered an error trying to encode the value for this line:\n%s\n\nThe error was: %s" (line, inst))
+ ascii_name = fields[2]
+ country_id = fields[8]
+ admin1 = fields[10]
+ admin2 = fields[11]
+ admin3 = fields[12]
+ admin4 = fields[13]
+ admin1_id, admin2_id, admin3_id = [None] * 3
+ if admin1:
+ try:
+ admin1_id = self.admin1_codes[country_id][admin1]
+ except KeyError:
+ pass
+ if admin2:
+ try:
+ admin2_id = self.admin2_codes[country_id][admin1][admin2]
+ except KeyError:
+ pass
+ if admin3:
+ try:
+ admin3_id = self.admin3_codes[country_id][admin1][admin2][admin3]
+ except KeyError:
+ pass
+ try:
+ self.cursor.execute(u'INSERT INTO admin4_code (country_id, admin1_id, admin2_id, admin3_id, geoname_id, code, name, ascii_name) VALUES (%s, %s, %s, %s, %s, %s, %s, %s)', (country_id, admin1_id, admin2_id, admin3_id, geoname_id, admin4, name, ascii_name))
+ except Exception, e:
+ if 'Duplicate' in str(e):
+ #print "I think the data was already entered."
+ return True
+ self.handle_exception(e, line)
+
+ self.admin4_codes.setdefault(country_id, {})
+ self.admin4_codes[country_id].setdefault(admin1, {})
+ self.admin4_codes[country_id][admin1].setdefault(admin2, {})
+ self.admin4_codes[country_id][admin1][admin2].setdefault(admin3, {})
+ self.admin4_codes[country_id][admin1][admin2][admin3][admin4] = self.last_row_id('admin4_code', 'id')
+
+ line = fd.readline()[:-1]
+
+ fd.close()
+ print '%d fourth level administrative divisions imported' % self.table_count('admin4_code')
+
+
+ def import_geonames(self):
+ print 'Importing geonames (this is going to take a while)'
+ fd = open('allCountries.txt')
+ line = fd.readline()[:-1]
+ while line:
+ fields = line.split('\t')
+ id, name, ascii_name = fields[:3]
+ latitude, longitude, fclass, fcode, country_id, cc2 = fields[4:10]
+ population, elevation, gtopo30 = fields[14:17]
+ if fclass != 'P': #only import populated places!
+ line = fd.readline()[:-1]
+ continue
+ moddate = fields[18]
+ if elevation == '':
+ elevation = 0
+ try:
+ timezone_id = self.time_zones[fields[17]]
+ except KeyError:
+ timezone_id = None
+ #XXX
+ try:
+ name = unicode(name,'utf-8')
+ except Exception, inst:
+ raise Exception("Encountered an error trying to encode the value for this line:\n%s\n\nThe error was: %s" (line, inst))
+ admin1 = fields[10]
+ admin2 = fields[11]
+ admin3 = fields[12]
+ admin4 = fields[13]
+ admin1_id, admin2_id, admin3_id, admin4_id = [None] * 4
+
+ if admin1:
+ try:
+ admin1_id = self.admin1_codes[country_id][admin1]
+ except KeyError:
+ pass
+
+ if admin2:
+ try:
+ admin2_id = self.admin2_codes[country_id][admin1][admin2]
+ except KeyError:
+ pass
+
+ if admin3:
+ try:
+ admin3_id = self.admin3_codes[country_id][admin1][admin2][admin3]
+ except KeyError:
+ pass
+
+ if admin4:
+ try:
+ admin4_id = self.admin4_codes[country_id][admin1][admin2][admin3][admin4]
+ except KeyError:
+ pass
+ try:
+ self.cursor.execute(u'INSERT INTO geoname (id, name, ascii_name, latitude, longitude, fclass, fcode, country_id, cc2, admin1_id, admin2_id, admin3_id, admin4_id, population, elevation, gtopo30, timezone_id, moddate) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)', (id, name, ascii_name, latitude, longitude, fclass, fcode, country_id, cc2, admin1_id, admin2_id, admin3_id, admin4_id, population, elevation, gtopo30, timezone_id, moddate))
+ except Exception, e:
+ if 'Duplicate' in str(e):
+ #print "I think the data was already entered."
+ return True
+ self.handle_exception(e, line)
+
+ line = fd.readline()[:-1]
+ fd.close()
+
+ print '%d geonames imported' % self.table_count('geoname')
+
+ def import_all(self):
+ self.pre_import()
+ self.begin()
+ self.import_fcodes()
+ self.commit()
+ self.begin()
+ self.import_language_codes()
+ self.commit()
+ self.begin()
+ self.import_alternate_names()
+ self.commit()
+ self.begin()
+ self.import_time_zones()
+ self.commit()
+ self.begin()
+ self.import_continent_codes()
+ self.commit()
+ self.begin()
+ self.import_countries()
+ self.commit()
+ self.begin()
+ self.import_first_level_adm()
+ self.commit()
+ self.begin()
+ self.import_second_level_adm()
+ self.commit()
+ self.begin()
+ self.import_third_level_adm()
+ self.commit()
+ self.begin()
+ self.import_fourth_level_adm()
+ self.commit()
+ self.begin()
+ self.import_geonames()
+ self.commit()
+ self.post_import()
+
+class PsycoPg2Importer(GeonamesImporter):
+ def table_count(self, table):
+ return 0
+
+ def pre_import(self):
+ self.end_stmts = []
+ import re
+ from django.core.management.color import no_style
+ from django.core.management.sql import sql_all
+ from django.db import models
+ sys.path.append('../')
+ sys.path.append('../../')
+
+ alter_re = re.compile('^ALTER TABLE "(\w+)" ADD CONSTRAINT (\w+).*', re.I)
+ alter_action = 'ALTER TABLE "\g<1>" DROP CONSTRAINT "\g<2>"'
+ index_re = re.compile('^CREATE INDEX "(\w+)".*', re.I)
+ index_action = 'DROP INDEX "\g<1>"'
+ table_re = re.compile('^CREATE TABLE "(\w+)".*', re.I)
+ references_re = re.compile('"(\w+)".*?REFERENCES "(\w+)" \("(\w+)"\) DEFERRABLE INITIALLY DEFERRED')
+ references_action = 'ALTER TABLE "%(table)s" DROP CONSTRAINT "%(table)s_%(field)s_fkey"'
+ references_stmt = 'ALTER TABLE "%(table)s" ADD CONSTRAINT "%(table)s_%(field)s_fkey" FOREIGN KEY ("%(field)s") ' \
+ 'REFERENCES "%(reftable)s" ("%(reffield)s") DEFERRABLE INITIALLY DEFERRED'
+ sql = sql_all(models.get_app('geonames'), no_style(), connections[DEFAULT_DB_ALIAS])
+ for stmt in sql:
+ if alter_re.search(stmt):
+ self.cursor.execute(alter_re.sub(alter_action, stmt))
+ self.end_stmts.append(stmt)
+ elif index_re.search(stmt):
+ self.cursor.execute(index_re.sub(index_action, stmt))
+ self.end_stmts.append(stmt)
+ elif table_re.search(stmt):
+ table = table_re.search(stmt).group(1)
+ for m in references_re.findall(stmt):
+ self.cursor.execute(references_action % \
+ {
+ 'table': table,
+ 'field': m[0],
+ 'reftable': m[1],
+ 'reffield': m[2],
+ })
+ self.end_stmts.append(references_stmt % \
+ {
+ 'table': table,
+ 'field': m[0],
+ 'reftable': m[1],
+ 'reffield': m[2],
+ })
+
+ self.cursor.execute('COMMIT')
+
+ def post_import(self):
+ print 'Enabling constraings and generating indexes (be patient, this is the last step)'
+ self.insert_dummy_records()
+ for stmt in self.end_stmts:
+ self.cursor.execute(stmt)
+ self.commit()
+
+ def insert_dummy_records(self):
+ self.cursor.execute("UPDATE geoname SET country_id='' WHERE country_id IN (' ', ' ')")
+ self.cursor.execute("DELETE FROM country WHERE geoname_id=6295630 or iso_numeric=-1")
+ self.cursor.execute("DELETE FROM continent WHERE geoname_id=6295630")
+ self.cursor.execute("INSERT INTO country (iso_alpha2, iso_alpha3, iso_numeric, fips_code, name, capital, area, population, continent_id, tld, currency_code, currency_name, phone_prefix, postal_code_fmt, postal_code_re, languages, geoname_id) VALUES ('', '', -1, '', 'No country', 'No capital', 0, 0, '', '', '', '', '', '', '', '', 6295630)")
+ self.cursor.execute("INSERT INTO continent VALUES('', 'No continent', 6295630)")
+
+ def begin(self):
+ self.cursor.execute('BEGIN')
+
+ def commit(self):
+ self.cursor.execute('COMMIT')
+
+ def get_db_conn(self):
+ import psycopg2
+ conn_params = 'dbname=%s ' % self.db
+ if self.host:
+ conn_params += 'host=%s ' % self.host
+ if self.user:
+ conn_params += 'user=%s ' % self.user
+ if self.password:
+ conn_params += 'password=%s' % self.password
+
+ self.conn = psycopg2.connect(conn_params)
+ self.cursor = self.conn.cursor()
+
+ def last_row_id(self, table=None, pk=None):
+ self.cursor.execute("SELECT CURRVAL('\"%s_%s_seq\"')" % (table, pk))
+ return self.cursor.fetchone()[0]
+
+ def set_import_date(self):
+ self.cursor.execute('INSERT INTO geonames_update (updated_date) VALUES ( CURRENT_DATE AT TIME ZONE \'UTC\')')
+
+class MySQLImporter(GeonamesImporter):
+ def table_count(self, table):
+ return 0
+
+ def import_file(self, tablename, filename):
+ import re
+ if re.search(r'[^\w\.]',tablename):
+ raise Exception("Illegal tablename: %s" % tablename)
+ try:
+ open(filename)
+ except:
+ raise Exception("Bad file.")
+ fullpath = "%s/%s" % (os.getcwd(),filename)
+ print "LOAD DATA INFILE '%(filename)s' IGNORE INTO TABLE `%(tablename)s` CHARACTER SET utf8" % {'tablename':tablename, 'filename':fullpath}
+ self.cursor.execute("LOAD DATA INFILE '%(filename)s' IGNORE INTO TABLE `%(tablename)s`" % {'tablename':tablename, 'filename':fullpath})
+
+ def pre_import(self):
+ self.end_stmts = []
+ import re
+ from django.core.management.color import no_style
+ from django.core.management.sql import sql_all
+ from django.db import models
+ sys.path.append('../')
+ sys.path.append('../../')
+
+ alter_re = re.compile('^ALTER TABLE "(\w+)" ADD CONSTRAINT (\w+).*', re.I)
+ alter_action = 'ALTER TABLE "\g<1>" DROP CONSTRAINT "\g<2>"'
+ index_re = re.compile('^CREATE INDEX "(\w+)".*', re.I)
+ index_action = 'DROP INDEX "\g<1>"'
+ table_re = re.compile('^CREATE TABLE "(\w+)".*', re.I)
+ references_re = re.compile('"(\w+)".*?REFERENCES "(\w+)" \("(\w+)"\) DEFERRABLE INITIALLY DEFERRED')
+ references_action = 'ALTER TABLE "%(table)s" DROP CONSTRAINT "%(table)s_%(field)s_fkey"'
+ references_stmt = 'ALTER TABLE "%(table)s" ADD CONSTRAINT "%(table)s_%(field)s_fkey" FOREIGN KEY ("%(field)s") ' \
+ 'REFERENCES "%(reftable)s" ("%(reffield)s")'
+ sql = sql_all(models.get_app('geonames'), no_style(), connections[DEFAULT_DB_ALIAS])
+ for stmt in sql:
+ if alter_re.search(stmt):
+ self.cursor.execute(alter_re.sub(alter_action, stmt))
+ self.end_stmts.append(stmt)
+ elif index_re.search(stmt):
+ self.cursor.execute(index_re.sub(index_action, stmt))
+ self.end_stmts.append(stmt)
+ elif table_re.search(stmt):
+ table = table_re.search(stmt).group(1)
+ for m in references_re.findall(stmt):
+ self.cursor.execute(references_action % \
+ {
+ 'table': table,
+ 'field': m[0],
+ 'reftable': m[1],
+ 'reffield': m[2],
+ })
+ self.end_stmts.append(references_stmt % \
+ {
+ 'table': table,
+ 'field': m[0],
+ 'reftable': m[1],
+ 'reffield': m[2],
+ })
+
+ self.cursor.execute('COMMIT')
+
+ def post_import(self):
+ print 'Enabling constraings and generating indexes (be patient, this is the last step)'
+ self.insert_dummy_records()
+ for stmt in self.end_stmts:
+ self.cursor.execute(stmt)
+ self.commit()
+
+ def insert_dummy_records(self):
+ self.cursor.execute("DELETE FROM country WHERE geoname_id=6295630 or iso_numeric=-1")
+ self.cursor.execute("DELETE FROM continent WHERE geoname_id=6295630")
+ self.cursor.execute("UPDATE geoname SET country_id='' WHERE country_id IN (' ', ' ')")
+ self.cursor.execute("INSERT INTO country VALUES ('', '', -1, '', 'No country', 'No capital', 0, 0, '', '', '', '', '', '', '', '', 6295630)")
+ self.cursor.execute("INSERT INTO continent VALUES('', 'No continent', 6295630)")
+
+ def begin(self):
+ self.cursor.execute('BEGIN')
+
+ def commit(self):
+ self.cursor.execute('COMMIT')
+
+ def get_db_conn(self):
+ import MySQLdb
+ conn_params = {}
+ conn_params['db'] = self.db
+ if self.host:
+ conn_params['host'] = self.host
+ if self.user:
+ conn_params['user'] = self.user
+ if self.password:
+ conn_params['passwd'] = self.password
+
+ conn_params['use_unicode'] = True
+
+ self.conn = MySQLdb.connect(**conn_params)
+ self.conn.set_character_set('utf8')
+ self.cursor = self.conn.cursor()
+ self.cursor.execute('SET NAMES utf8;')
+ self.cursor.execute('SET CHARACTER SET utf8;')
+ self.cursor.execute('SET character_set_connection=utf8;')
+
+ def last_row_id(self, table=None, pk=None):
+ self.cursor.execute("SELECT MAX(%s) from %s" % (pk,table))
+ return self.cursor.fetchone()[0]
+
+ def set_import_date(self):
+ self.cursor.execute('INSERT INTO geonames_update (updated_date) VALUES ( Now() )')
+
+IMPORTERS = {
+ 'postgresql_psycopg2': PsycoPg2Importer,
+ 'django.contrib.gis.db.backends.mysql': MySQLImporter,
+}
+
+def main(options):
+ options = {'tmpdir':'geonames_temp'} #TODO: Make this a proper option
+ try:
+ importer = IMPORTERS[(settings.DATABASES and settings.DATABASES['default']['ENGINE']) or settings.DATABASE_ENGINE]
+ except KeyError:
+ print 'Sorry, database engine "%s" is not supported' % \
+ settings.DATABASE_ENGINE
+ sys.exit(1)
+
+ try:
+ imp = importer(host=settings.DATABASES['default'].get('HOST',None),
+ user=settings.DATABASES['default']['USER'],
+ password=settings.DATABASES['default']['PASSWORD'],
+ db=settings.DATABASES['default']['NAME'],
+ tmpdir=options['tmpdir'])
+ print "I was a success"
+ except AttributeError:
+ imp = importer(host=settings.DATABASE_HOST,
+ user=settings.DATABASE_USER,
+ password=settings.DATABASE_PASSWORD,
+ db=settings.DATABASE_NAME,
+ tmpdir=options['tmpdir'])
+
+ imp.fetch()
+ imp.get_db_conn()
+ imp.import_all()
+ imp.set_import_date()
+ imp.cleanup()
+
+class Command(BaseCommand):
+ help = "Geonames import command."
+
+ def handle(self, *args, **options):
+ main(options)
View
460 build/lib/geonames/models.py
@@ -0,0 +1,460 @@
+# This file is part of Django-Geonames
+# Copyright (c) 2008, Alberto Garcia Hierro
+# See LICENSE file for details
+
+from math import sin, cos, acos, radians
+
+from django.core.cache import cache
+from django.db import connection
+from django.contrib.gis.db import models
+from django.utils.translation import ugettext, get_language
+from django.conf import settings
+
+from geonames.decorators import stored_property, cache_set
+
+GLOBE_GEONAME_ID = 6295630
+
+
+def translate_geoname(g, lang):
+ cursor = connection.cursor()
+ cursor.execute('''SELECT name FROM alternate_name WHERE language='%(lang)s' \
+ AND geoname_id = %(id)d AND preferred=TRUE UNION SELECT name \
+ FROM alternate_name WHERE language='%(lang)s' AND geoname_id = %(id)d LIMIT 1''' % \
+ { 'lang': lang, 'id': g.id })
+
+ try:
+ return cursor.fetchone()[0]
+ except TypeError:
+ return g.name
+
+def get_geo_translate_func():
+ try:
+ cnf = settings.GEONAMES_TRANSLATION_METHOD
+ except AttributeError:
+ cnf = 'NOOP'
+
+ if cnf == 'NOOP':
+ return (lambda x: x.name)
+
+ if cnf == 'STATIC':
+ lang = settings.LANGUAGE_CODE.split('-')[0]
+ if lang == 'en':
+ return (lambda x: x.name)
+
+ def geo_translate(self):
+ key = 'Geoname_%s_i18n_name' % self.id
+ return cache.get(key) or cache_set(key, translate_geoname(self, lang))
+
+ return geo_translate
+
+ if cnf == 'DYNAMIC':
+ def geo_translate(self):
+ lang = get_language()
+ key = 'Geoname_%s_%s_i18n_name' % (self.id, lang)
+ return cache.get(key) or cache_set(key, translate_geoname(self, lang))
+
+ return geo_translate
+
+
+ raise ValueError('Unknown value for GEONAMES_TRANSLATION_METHOD: "%s"' % cnf)
+
+geo_translate_func = get_geo_translate_func()
+
+class GeonameGISHelper(object):
+ def near_point(self, latitude, longitude, kms, order):
+ raise NotImplementedError
+
+class MySQLGeonameGISHelper(GeonameGISHelper):
+
+ def near_point(self, latitude, longitude, kms, order):
+ from math import degrees, radians
+ EARTH_RADIUS=3956.547
+ dist = "fn_distance_cosine_km(%f, %f, latitude, longitude)" % (latitude, longitude)
+ if order:
+ order_by = ['distance']
+ else:
+ order_by = None
+ # fast approximation to eliminate points outside of our search
+ max_lat = float(latitude) + degrees(kms/EARTH_RADIUS);
+ min_lat = float(latitude) - degrees(kms/EARTH_RADIUS);
+ max_long = float(longitude) + degrees(kms/EARTH_RADIUS/cos(radians(latitude)));
+ min_long = float(longitude) - degrees(kms/EARTH_RADIUS/cos(radians(latitude)));
+ near_objects = Geoname.objects.all()
+ near_objects = near_objects.filter(latitude__gte=str(min_lat), longitude__gte=str(min_long))
+ near_objects = near_objects.filter(latitude__lte=str(max_lat), longitude__lte=str(max_long))
+ if near_objects.count():
+ near_objects = near_objects.extra(
+ select = { 'distance':dist },
+ where = ["%(dist)s < %(kms)d" % { 'dist':dist, 'kms':kms } ],
+ order_by = order_by
+ )
+ return near_objects
+
+GIS_HELPERS = {
+ 'django.contrib.gis.db.backends.mysql': MySQLGeonameGISHelper
+}
+
+try:
+ GISHelper = GIS_HELPERS[(settings.DATABASES and settings.DATABASES['default']['ENGINE']) or settings.DATABASE_ENGINE]()
+except (KeyError,AttributeError):
+ print 'Sorry, your database backend is not supported by the Geonames application'
+
+class Geoname(models.Model):
+ id = models.IntegerField(primary_key=True)
+ name = models.CharField(max_length=200, db_index=True)
+ ascii_name = models.CharField(max_length=200)
+ latitude = models.DecimalField(max_digits=20, decimal_places=17)
+ longitude = models.DecimalField(max_digits=20, decimal_places=17)
+ point = models.PointField(null=True, blank=True)
+ fclass = models.CharField(max_length=1, db_index=True)
+ fcode = models.ForeignKey('FeatureCode', to_field='code', db_column='fcode', db_index=True)
+ country = models.ForeignKey('Country', db_index=True, related_name='geoname_set')
+ cc2 = models.CharField(max_length=60)
+ admin1 = models.ForeignKey('Admin1Code', null=True, related_name='geoname_set', db_index=True)
+ admin2 = models.ForeignKey('Admin2Code', null=True, related_name='geoname_set', db_index=True)
+ admin3 = models.ForeignKey('Admin3Code', null=True, related_name='geoname_set', db_index=True)
+ admin4 = models.ForeignKey('Admin4Code', null=True, related_name='geoname_set', db_index=True)
+ population = models.IntegerField()
+ elevation = models.IntegerField()
+ gtopo30 = models.IntegerField()
+ timezone = models.ForeignKey('Timezone', null=True)
+ moddate = models.DateField()
+
+ objects = models.GeoManager()
+
+ class Meta:
+ db_table = 'geoname'
+
+ def __unicode__(self):
+ return self.name
+
+ def save(self, *args, **kwargs):
+ self.point = 'POINT(%s %s)' % (self.longitude, self.latitude)
+ super(Geoname, self).save(*args, **kwargs)
+
+ @stored_property
+ def i18n_name(self):
+ return geo_translate_func(self)
+
+ @stored_property
+ def admin1_i18n_name(self):
+ if self.fcode in ('', 'CONT', 'PCLI'):
+ return u''
+ try:
+ return self.admin1.geoname.i18n_name
+ except (Admin1Code.DoesNotExist, Geoname.DoesNotExist):
+ return u''
+
+ @stored_property
+ def fcode_name(self):
+ try:
+ return ugettext(FeatureCode.objects.get(pk=self.fcode).name)
+ except FeatureCode.DoesNotExist:
+ return u''
+
+ @stored_property
+ def country_name(self):
+ try:
+ return self.country.__unicode__()
+ except Country.DoesNotExist:
+ return u''
+
+ @stored_property
+ def country_i18n_name(self):
+ try:
+ return self.country.geoname.i18n_name
+ except models.Model.DoesNotExist:
+ return u''
+
+ @stored_property
+ def parent(self):
+ if self.id == GLOBE_GEONAME_ID:
+ return None
+ return self.get_parent
+
+ def get_parent(self):
+
+ if self.fcode == 'CONT':
+ return Geoname.globe()
+
+ if self.fcode.startswith('PCL'):
+ g_list = [self.country.continent]
+ elif self.fcode in ('ADM1', 'ADMD'):
+ g_list = [self.country, self.country.continent]
+ elif self.fcode == 'ADM2':
+ g_list = [self.admin1, self.country, self.country.continent]
+ elif self.fcode == 'ADM3':
+ g_list = [self.admin2, self.admin1, self.country, self.country.continent]
+ elif self.fcode == 'ADM4':
+ g_list = [self.admin3, self.admin2, self.admin1, self.country, self.country.continent]
+ else:
+ g_list = [self.admin4, self.admin3, self.admin2, self.admin1, self.country, self.country.continent]
+
+ for g in g_list:
+ try:
+ if g.geoname_id != self.id:
+ return g.geoname
+ except AttributeError:
+ pass
+
+ return None
+
+ @stored_property
+ def hierarchy(self):
+ hier = []
+ parent = self.parent
+ while parent:
+ hier.append(parent)
+ parent = parent.parent
+
+ return hier
+
+ def get_children(self):
+ if self.id == GLOBE_GEONAME_ID:
+ return Geoname.objects.filter(id__in=[x['geoname'] for x in Continent.objects.values('geoname')])
+
+ if self.fcode == 'CONT':
+ return Geoname.objects.filter(id__in=[x['geoname'] for x in Continent.objects.get(geoname=self.id).country_set.values('geoname')])
+
+ if self.fclass != 'A':
+ return Geoname.objects.none()
+
+ try:
+ if self.fcode.startswith('PCL'):
+ s_list = [self.country.geoname_set.filter(fcode=code) for code in ('ADM1', 'ADMD', 'ADM2', 'ADM3', 'ADM4')] + [self.country.geoname_set.filter(fclass='P')]
+ elif self.fcode == 'ADM1':
+ s_list = [self.admin1.geoname_set.filter(fcode=code) for code in ('ADM2', 'ADM3', 'ADM4')] + [self.admin1.geoname_set.filter(fclass='P')]
+ elif self.fcode == 'ADM2':
+ s_list = [self.admin2.geoname_set.filter(fcode=code) for code in ('ADM3', 'ADM4')] + [self.admin2.geoname_set.filter(fclass='P')]
+ elif self.fcode == 'ADM3':
+ s_list = [self.admin3.geoname_set.filter(fcode='ADM4'), self.admin3.geoname_set.filter(fclass='P')]
+ elif self.fcode == 'ADM4':
+ s_list = [self.admin4.geoname_set.filter(fclass='P')]
+ else:
+ return Geoname.objects.none()
+
+ except AttributeError:
+ return Geoname.objects.none()
+
+ for qs in s_list:
+ if qs.count():
+ return qs
+
+ return Geoname.objects.none()
+
+ @stored_property
+ def children(self):
+ cset = self.get_children()
+ l = list(cset or [])
+ l.sort(cmp=lambda x,y: cmp(x.i18n_name, y.i18n_name))
+ return l
+
+ @classmethod
+ def biggest(cls, lset):
+ codes = [ '', 'CONT', 'PCLI', 'ADM1', 'ADM2', 'ADM3', 'ADM4', 'PPL']
+ for c in codes:
+ for item in lset:
+ if item.fcode == c:
+ return item
+
+ try:
+ return lset[0]
+ except IndexError:
+ return None
+
+ @classmethod
+ def globe(cls):
+ return cls.objects.get(pk=GLOBE_GEONAME_ID)
+
+ def is_globe(self):
+ return self.id == GLOBE_GEONAME_ID
+
+ def contains(self, child):
+ if self.is_globe():
+ return True
+ try:
+ if self.fcode == 'CONT':
+ return child.country.continent.geoname == self
+ if self.fcode in ('PCLI', 'PCLD'):
+ return child.country_id == self.country_id
+ if self.fcode == 'ADM1':
+ return self.admin1_id == child.admin1_id
+ if self.fcode == 'ADM2':
+ return self.admin2_id == child.admin2_id
+ if self.fcode == 'ADM3':
+ return self.admin3_id == child.admin3_id
+ if self.fcode == 'ADM4':
+ return self.admin4_id == child.admin4_id
+ except Country.DoesNotExist:
+ return False
+
+ return False
+
+ def distance(self, other):
+ return Geoname.distance_points(self.latitude, self.longitude, other.latitude, other.longitude)
+
+ def near_me(self, kms=20, order=True):
+ return Geoname.near_point(self.latitude, self.longitude, kms=kms, order=order).exclude(pk=self.pk)
+
+ @staticmethod
+ def select_fields():
+ return 'id, name, ascii_name, latitude, longitude, fclass, fcode,' \
+ ' country_id, cc2, admin1_id, admin2_id, admin3_id, ' \
+ ' admin4_id, population, elevation, gtopo30, timezone_id, moddate'
+
+ @classmethod
+ def distance_points(cls, lat1, lon1, lat2, lon2, is_rad=False):
+ if not is_rad:
+ lat1, lon1, lat2, lon2 = map(lambda x: radians(float(x)), (lat1, lon1, lat2, lon2))
+ return 6378.7 * acos(sin(lat1) * sin(lat2) + cos(lat1) * cos(lat2) * cos(lon2 - lon1))
+
+ @staticmethod
+ def near_point(latitude, longitude, kms=20, order=True):
+ return GISHelper.near_point(latitude, longitude, kms, order)
+
+class GeonameAlternateName(models.Model):
+ id = models.IntegerField(primary_key=True)
+ geoname = models.ForeignKey(Geoname, related_name='altnames', db_index=True)
+ language = models.CharField(max_length=7)
+ name = models.CharField(max_length=200)
+ preferred = models.BooleanField()
+ short = models.BooleanField()
+
+ class Meta:
+ db_table = 'alternate_name'
+
+ def __unicode__(self):
+ return "%s -> %s" % (self.name,self.geoname.name)
+
+class Continent(models.Model):
+ code = models.CharField(max_length=2, primary_key=True)
+ name = models.CharField(max_length=20)
+ geoname = models.ForeignKey(Geoname, unique=True)
+
+ class Meta:
+ db_table = 'continent'
+
+ def __unicode__(self):
+ return self.name
+
+class Country(models.Model):
+ iso_alpha2 = models.CharField(max_length=2, primary_key=True)
+ iso_alpha3 = models.CharField(max_length=3, unique=True)
+ iso_numeric = models.IntegerField(unique=True)
+ fips_code = models.CharField(max_length=3)
+ name = models.CharField(max_length=200)
+ capital = models.CharField(max_length=200)
+ area = models.FloatField()
+ population = models.IntegerField()
+ continent = models.ForeignKey(Continent, db_index=True)
+ tld = models.CharField(max_length=4, null=True)
+ currency_code = models.CharField(max_length=3)
+ currency_name = models.CharField(max_length=16, null=True)
+ phone_prefix = models.CharField(max_length=16, null=True)
+ postal_code_fmt = models.CharField(max_length=64, null=True)
+ postal_code_re = models.CharField(max_length=256, null=True)
+ languages = models.CharField(max_length=200)
+ geoname = models.ForeignKey(Geoname, related_name='this_country')
+ neighbours = models.ManyToManyField('self')
+
+ class Meta:
+ db_table = 'country'
+
+ def __unicode__(self):
+ return self.name
+
+class Language(models.Model):
+ iso_639_3 = models.CharField(max_length=4, primary_key=True)
+ iso_639_2 = models.CharField(max_length=50)
+ iso_639_1 = models.CharField(max_length=50)
+ language_name = models.CharField(max_length=200)
+
+ class Meta:
+ db_table = 'iso_language'
+
+class Admin1Code(models.Model):
+ country = models.ForeignKey(Country, db_index=True)
+ geoname = models.ForeignKey(Geoname, db_index=True)
+ code = models.CharField(max_length=5)
+ name = models.TextField()
+ ascii_name = models.TextField()
+ geom = models.GeometryField()
+ class Meta:
+ db_table = 'admin1_code'
+
+ def __unicode__(self):
+ return self.name
+
+class Admin2Code(models.Model):
+ country = models.ForeignKey(Country, db_index=True)
+ admin1 = models.ForeignKey(Admin1Code, null=True)
+ geoname = models.ForeignKey(Geoname, db_index=True)
+ code = models.CharField(max_length=30)
+ name = models.TextField()
+ ascii_name = models.TextField()
+ geom = models.GeometryField()
+ class Meta:
+ db_table = 'admin2_code'
+ def __unicode__(self):
+ return self.name
+
+
+class Admin3Code(models.Model):
+ country = models.ForeignKey(Country, db_index=True)
+ admin1 = models.ForeignKey(Admin1Code, null=True, db_index=True)
+ admin2 = models.ForeignKey(Admin2Code, null=True, db_index=True)
+ geoname = models.ForeignKey(Geoname, db_index=True)
+ code = models.CharField(max_length=30)
+ name = models.TextField()
+ ascii_name = models.TextField()
+ geom = models.GeometryField()
+ class Meta:
+ db_table = 'admin3_code'
+ def __unicode__(self):
+ return self.name
+
+
+class Admin4Code(models.Model):
+ country = models.ForeignKey(Country)
+ admin1 = models.ForeignKey(Admin1Code, null=True, db_index=True)
+ admin2 = models.ForeignKey(Admin2Code, null=True, db_index=True)
+ admin3 = models.ForeignKey(Admin3Code, null=True, db_index=True)
+ geoname = models.ForeignKey(Geoname, db_index=True)
+ code = models.CharField(max_length=30)
+ name = models.TextField()
+ ascii_name = models.TextField()
+ geom = models.GeometryField()
+ class Meta:
+ db_table = 'admin4_code'
+ def __unicode__(self):
+ return self.name
+
+
+class FeatureCode(models.Model):
+ code = models.CharField(max_length=7, primary_key=True)
+ fclass = models.CharField(max_length=1)
+ name = models.CharField(max_length=200)
+ description = models.TextField()
+
+ class Meta:
+ db_table = 'feature_code'
+ def __unicode__(self):
+ return self.name
+
+
+class Timezone(models.Model):
+ name = models.CharField(max_length=200)
+ gmt_offset = models.DecimalField(max_digits=4, decimal_places=2)
+ dst_offset = models.DecimalField(max_digits=4, decimal_places=2)
+
+ class Meta:
+ db_table = 'time_zone'
+ def __unicode__(self):
+ return self.name
+
+
+class GeonamesUpdate(models.Model):
+ updated_date = models.DateField()
+
+ class Meta:
+ db_table = 'geonames_update'
View
BIN geonames/.DS_Store
Binary file not shown.
View
0 geonames/__init__.py
No changes.
View
68 geonames/decorators.py
@@ -0,0 +1,68 @@
+# -*- coding: utf-8 -*-
+
+# The contents of this file are subject to the Common Public Attribution License
+# Version 1.0 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.cpal-dev.com/cpaldemoopen/license. The License is based on the
+# Mozilla Public License Version 1.1 but Sections 14 and 15 have been added to cover
+# use of software over a computer network and provide for limited attribution for
+# the Original Developer. In addition, Exhibit A has been modified to be consistent
+# with Exhibit B.
+
+# Software distributed under the License is distributed on an "AS IS" basis, WITHOUT
+# WARRANTY OF ANY KIND, either express or implied. See the License for the specific
+# anguage governing rights and limitations under the License.
+
+# The Original Code is the byNotes project
+
+# The Initial Developer of the Original Code is Alberto García Hierro
+# The Original Developer is Alberto García Hierro
+
+# All portions of the code written by Alberto García Hierro are
+# Copyright (C) 2008 Alberto García Hierro
+# All Rights Reserved.
+
+from django.core.cache import cache
+
+def cache_set(key, value):
+ cache.set(key, value)
+ return value
+
+def _cached(func):
+ def cached_func(self):
+ key = 'cached_property_%s_%s_%s' % \
+ (self.__class__.__name__, func.__name__, self.pk)
+ val = cache.get(key)
+ return cache_set(key, func(self)) if val is None else val
+
+ cached_func.__name__ = func.__name__
+ return cached_func
+
+def cached_property(func):
+ return property(_cached(func))
+
+def _stored(func):
+ key = '_cached_%s' % func.__name__
+ def stored_func(self):
+ if not hasattr(self, key):
+ setattr(self, key, func(self))
+ return getattr(self, key)
+
+ stored_func.__name__ = func.__name__
+ return stored_func
+
+def stored_property(func):
+ return property(_stored(func))
+
+def full_cached_property(func):
+ return stored_property(_cached(func))
+
+def cached_method(func):
+ def cached_func(self, *args, **kwargs):
+ key = 'cached_method_%s_%s_%s_%s_%s' % \
+ (self.__class__.__name__, func.__name__, self.pk, hash(args),
+ hash(kwargs.iteritems()))
+ val = cache.get(key)
+ return cache_set(key, func(self, *args, **kwargs)) if val is None else val
+
+ return cached_func
View
33 geonames/fcodes.py
@@ -0,0 +1,33 @@
+# This file is part of Django-Geonames
+# Copyright (c) 2008, Alberto Garcia Hierro
+# See LICENSE file for details
+
+FCODES = [
+ ugettext_noop('first-order administrative division'),
+ ugettext_noop('second-order administrative division'),
+ ugettext_noop('third-order administrative division'),
+ ugettext_noop('fourth-order administrative division'),
+ ugettext_noop('administrative division'),
+ ugettext_noop('leased area'),
+ ugettext_noop('political entity'),
+ ugettext_noop('dependent political entity'),
+ ugettext_noop('freely associated state'),
+ ugettext_noop('independent political entity'),
+ ugettext_noop('section of independent political entity'),
+ ugettext_noop('semi-independent political entity'),
+ ugettext_noop('parish'),
+ ugettext_noop('territory'),
+ ugettext_noop('zone'),
+ ugettext_noop('buffer zone'),
+ ugettext_noop('populated place'),
+ ugettext_noop('seat of a first-order administrative division'),
+ ugettext_noop('capital of a political entity'),
+ ugettext_noop('seat of government of a political entity'),
+ ugettext_noop('populated locality'),
+ ugettext_noop('abandoned populated place'),
+ ugettext_noop('religious populated place'),
+ ugettext_noop('populated places'),
+ ugettext_noop('destroyed populated place'),
+ ugettext_noop('section of populated place'),
+ ugettext_noop('israeli settlement'),
+]
View
16 geonames/gis/mysql/distance.sql
@@ -0,0 +1,16 @@
+
+DELIMITER $$
+
+DROP FUNCTION IF EXISTS `distance` $$
+CREATE FUNCTION distance(a POINT, b POINT) RETURNS double
+DETERMINISTIC
+COMMENT 'Spatial distance function using the great-circle distance formula (in km)'
+RETURN ( 6378.7
+ * acos(
+ sin( radians(X(a)) ) * sin( radians(X(b)) )
+ + cos( radians(X(a)) ) * cos( radians(X(b)) )
+* cos( radians(Y(b)) - radians(Y(a)) )
+)
+) $$
+
+DELIMITER ;
View
4 geonames/gis/mysql/indexes.sql
@@ -0,0 +1,4 @@
+--ALTER TABLE geoname ALTER COLUMN gpoint SET NOT NULL;
+--ALTER TABLE geoname ALTER COLUMN gpoint_meters SET NOT NULL;
+--CREATE INDEX geoname_gpoint ON geoname USING GIST ("gpoint" GIST_GEOMETRY_OPS);
+--CREATE INDEX geoname_gpoint_meters ON geoname USING GIST ("gpoint_meters" GIST_GEOMETRY_OPS);
View
5 geonames/gis/mysql/points.sql
@@ -0,0 +1,5 @@
+DROP TRIGGER IF EXISTS `geoname_point`;
+CREATE TRIGGER geoname_point BEFORE INSERT ON `geoname`
+ FOR EACH ROW
+ SET `NEW`.`point` = GeomFromText(CONCAT('POINT(',NEW.latitude, ' ', NEW.longitude, ')'));
+
View
40 geonames/gis/postgres/README
@@ -0,0 +1,40 @@
+If you need to process GIS data, follow these
+instructions. Note you don't need this
+if you are not going to do proximity searches.
+Note you need to do this before importing
+any data.
+
+First of all, you need postgis, which adds
+support for GIS data types to postgresql.
+Some distributions include packages, but if
+that's not the case for you, download
+it from http://postgis.refractions.net/
+and follow the installation instructions
+from the documentation.
+
+Next, execute the following commands
+(add username/password/host as needed):
+
+createlang plpgsql [yourdatabase]
+psql -d [yourdatabase] -f lwpostgis.sql
+psql -d [yourdatabase] -f spatial_ref_sys.sql
+
+(lwpostgis.sql and spatial_ref_sys.sql are distributed
+with postgis)
+
+This will create the spatial fields as well
+as a trigger for keeping them updated.
+psql -d [yourdatabase] < points.sql
+
+Finally, after importing the database, run:
+
+psql -d [yourdatabase] < indexes.sql
+
+This will create the indexes for the
+spatial fields.
+
+If you are doing only proximity searches,
+you should also cluster the table "geoname"
+according to the "geoname_point" index:
+(from psql console)
+CLUSTER geoname_point ON geoname;
View
4 geonames/gis/postgres/indexes.sql
@@ -0,0 +1,4 @@
+ALTER TABLE geoname ALTER COLUMN gpoint SET NOT NULL;
+ALTER TABLE geoname ALTER COLUMN gpoint_meters SET NOT NULL;
+CREATE INDEX geoname_gpoint ON geoname USING GIST ("gpoint" GIST_GEOMETRY_OPS);
+CREATE INDEX geoname_gpoint_meters ON geoname USING GIST ("gpoint_meters" GIST_GEOMETRY_OPS);
View
13 geonames/gis/postgres/points.sql
@@ -0,0 +1,13 @@
+/* this column is added for better performance in proximity searches */
+SELECT AddGeometryColumn('geoname', 'gpoint', 4326, 'POINT', 2);
+SELECT AddGeometryColumn('geoname', 'gpoint_meters', 32661, 'POINT', 2);
+CREATE FUNCTION geoname_points () RETURNS trigger AS $geoname_points$
+ BEGIN
+ NEW.gpoint = SetSRID(MakePoint(NEW.longitude, NEW.latitude), 4326);
+ NEW.gpoint_meters = Transform(NEW.gpoint, 32661);
+ RETURN NEW;
+ END
+$geoname_points$ LANGUAGE plpgsql;
+
+CREATE TRIGGER geoname_points BEFORE INSERT OR UPDATE ON geoname
+ FOR EACH ROW EXECUTE PROCEDURE geoname_points();
View
0 geonames/management/__init__.py
No changes.
View
0 geonames/management/commands/__init__.py
No changes.
View
733 geonames/management/commands/geonames_import.py
@@ -0,0 +1,733 @@
+import os
+import sys
+from optparse import OptionParser
+from warnings import filterwarnings
+from getpass import getpass
+from datetime import date
+from django.conf import settings
+from django.core.management.base import BaseCommand
+from django.db import connections, DEFAULT_DB_ALIAS
+
+FILES = [
+ 'http://download.geonames.org/export/dump/allCountries.zip',
+ 'http://download.geonames.org/export/dump/alternateNames.zip',
+ 'http://download.geonames.org/export/dump/admin1CodesASCII.txt',
+ 'http://download.geonames.org/export/dump/admin2Codes.txt',
+ 'http://download.geonames.org/export/dump/featureCodes_en.txt',
+ 'http://download.geonames.org/export/dump/timeZones.txt',
+ 'http://download.geonames.org/export/dump/countryInfo.txt',
+]
+
+CONTINENT_CODES = [
+ ('AF', 'Africa' , 6255146),
+ ('AS', 'Asia', 6255147),
+ ('EU', 'Europe', 6255148),
+ ('NA', 'North America', 6255149),
+ ('OC', 'Oceania', 6255151),
+ ('SA', 'South America', 6255150),
+ ('AN', 'Antarctica', 6255152),
+]
+
+class GeonamesImporter(object):
+ def __init__(self, host=None, user=None, password=None, db=None, tmpdir='tmp'):
+ self.user = user
+ self.password = password
+ self.db = db
+ self.host = host
+ self.conn = None
+ self.tmpdir = tmpdir
+ self.curdir = os.getcwd()
+ self.time_zones = {}
+ self.admin1_codes = {}
+ self.admin2_codes = {}
+ self.admin3_codes = {}
+ self.admin4_codes = {}
+
+ def pre_import(self):
+ pass
+
+ def post_import(self):
+ pass
+
+ def begin(self):
+ pass
+
+ def commit(self):
+ pass
+
+ def last_row_id(self, table=None, pk=None):
+ raise NotImplementedError('This is a generic importer, use one of the subclasses')
+
+ def get_db_conn(self):
+ raise NotImplementedError('This is a generic importer, use one of the subclasses')
+
+ def set_import_date(self):
+ raise NotImplementedError('This is a generic importer, use one of the subclasses')
+
+ def fetch(self):
+ try:
+ os.mkdir(self.tmpdir)
+ os.chdir(self.tmpdir)
+ except OSError:
+ os.chdir(self.tmpdir)
+ print 'Temporary directory %s exists, using already downloaded data'%self.tmpdir
+ return
+
+ for f in FILES:
+ if os.system('wget %s' % f) != 0:
+ print 'Error fetching %s' % os.path.basename(f)
+ sys.exit(1)
+
+ for f in ('allCountries.zip', 'alternateNames.zip'):
+ if os.system('unzip %s' % f) != 0:
+ print 'Error unzipping %s' % f
+ sys.exit(1)
+
+ def cleanup(self):
+ os.chdir(self.curdir)
+ for f in os.listdir(self.tmpdir):
+ os.unlink('%s/%s' % (self.tmpdir, f))
+ os.rmdir(self.tmpdir)
+
+ def handle_exception(self, e, line=None):
+ print e
+ sys.exit(1)
+
+ def table_count(self, table):
+ self.cursor.execute(u'SELECT COUNT(*) FROM %s' % table)
+ return self.cursor.fetchone()[0]
+
+ def import_fcodes(self):
+ print 'Importing feature codes'
+ fd = open('featureCodes_en.txt')
+ line = fd.readline()[:-1]
+ while line:
+ codes, name, desc = line.split('\t')
+ try:
+ fclass, code = codes.split('.')
+ except ValueError:
+ line = fd.readline()[:-1]
+ continue
+ try:
+ self.cursor.execute(u'INSERT INTO feature_code (code, fclass, name, description) VALUES (%s, %s, %s, %s)', (code, fclass, name, desc))
+ except Exception, e:
+ if 'Duplicate' in str(e):
+ #print "I think the data was already entered."
+ return True
+ self.handle_exception(e, line)
+
+ line = fd.readline()[:-1]
+ fd.close()
+ print '%d feature codes imported' % self.table_count('feature_code')
+
+ def import_language_codes(self):
+ print 'Importing language codes'
+ fd = open('iso-languagecodes.txt')
+ fd.readline()
+ line = fd.readline()[:-1]
+ while line:
+ fields = line.split('\t')
+ try:
+ self.cursor.execute(u'INSERT INTO iso_language (iso_639_3, iso_639_2, iso_639_1, language_name) VALUES (%s, %s, %s, %s)', fields)
+ except Exception, e:
+ if 'Duplicate' in str(e):
+ #print "I think the data was already entered."
+ return True
+ self.handle_exception(e)
+
+ line = fd.readline()[:-1]
+ fd.close()
+ print '%d language codes imported' % self.table_count('iso_language')
+
+ def import_alternate_names(self):
+ print 'Importing alternate names (this is going to take a while)'
+ if hasattr(self,'import_file'):
+ self.import_file('alternate_name','alternateNames.txt')
+ fd = open('alternateNames.txt')
+ line = fd.readline()[:-1]
+ while line:
+ id, geoname_id, lang, name, preferred, short = line.split('\t')
+ if preferred in ('', '0'):
+ preferred = False
+ else:
+ preferred = True
+ if short in ('', '0'):
+ short = False
+ else:
+ short = True
+ try:
+ self.cursor.execute(u'INSERT INTO alternate_name (id, geoname_id, language, name, preferred, short) VALUES (%s, %s, %s, %s, %s, %s)', (id, geoname_id, lang, name, preferred, short))
+ except Exception, e:
+ if 'Duplicate' in str(e):
+ #print "I think the data was already entered."
+ return True
+ self.handle_exception(e, line)
+ line = fd.readline()[:-1]
+ fd.close()
+ print '%d alternate names imported' % self.table_count('alternate_name')
+
+ def import_time_zones(self):
+ print 'Importing time zones'
+ fd = open('timeZones.txt')
+ fd.readline()
+ line = fd.readline()[:-1]
+ while line:
+ name, gmt, dst = line.split('\t')
+ try:
+ self.cursor.execute(u'INSERT INTO time_zone (name, gmt_offset, dst_offset) VALUES (%s, %s, %s)', (name, gmt, dst))
+ except Exception, e:
+ if 'Duplicate' in str(e):
+ #print "I think the data was already entered."
+ return True
+ self.handle_exception(e, line)
+
+ self.time_zones[name] = self.last_row_id('time_zone', 'id')
+ line = fd.readline()[:-1]
+ fd.close()
+ print '%d time zones imported' % self.table_count('time_zone')
+
+ def import_continent_codes(self):
+ for continent in CONTINENT_CODES:
+ try:
+ self.cursor.execute(u'INSERT INTO continent (code, name, geoname_id) VALUES (%s, %s, %s)', continent)
+ except Exception, e:
+ if 'Duplicate' in str(e):
+ #print "I think the data was already entered."
+ return True
+ self.handle_exception(e)
+ print '%d continent codes imported' % self.table_count('continent')
+
+ def import_countries(self):
+ print 'Importing countries'
+ fd = open('countryInfo.txt')
+ fd.readline()
+ line = fd.readline()[:-1]
+ while line:
+ if line[0] == '#' or line.startswith('ISO') or line.startswith('CS'):
+ line = fd.readline()[:-1]
+ continue
+ fields = line.split('\t')
+ #if len(fields) == 18:
+ # fields.append('')
+ fields[6] = fields[6].replace(',', '')
+ fields[7] = fields[7].replace(',', '')
+ if fields[6] == '':
+ fields[6] = 0
+ try:
+ self.cursor.execute(u'INSERT INTO country (iso_alpha2, iso_alpha3, iso_numeric, fips_code, name, capital, area, population, continent_id, tld, currency_code, currency_name, phone_prefix, postal_code_fmt, postal_code_re, languages, geoname_id) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)', fields[:17])
+ except Exception, e:
+ if 'Duplicate' in str(e):
+ #print "I think the data was already entered."
+ return True
+ self.handle_exception(e, line)
+ line = fd.readline()[:-1]
+ fd.close()
+ print '%d countries imported' % self.table_count('country')
+
+ def import_first_level_adm(self):
+ print 'Importing first level administrative divisions'
+ fd = open('admin1CodesASCII.txt')
+ line = fd.readline()[:-1]
+ while line:
+ country_and_code, name, ascii_name, geoname_id = line.split('\t')
+ country_id, code = country_and_code.split('.')
+ try:
+ name = unicode(name,'utf-8')
+ except Exception, inst:
+ raise Exception("Encountered an error trying to encode the value for this line:\n%s\n\nThe error was: %s" (line, inst))
+ try:
+ print(u'INSERT INTO admin1_code (country_id, geoname_id, code, name, ascii_name) VALUES (%s, %s, %s, %s, %s)' % (country_id, geoname_id, code, name, ascii_name))
+ self.cursor.execute(u'INSERT INTO admin1_code (country_id, geoname_id, code, name, ascii_name) VALUES (%s, %s, %s, %s, %s)', (country_id, geoname_id, code, name, ascii_name))
+ except Exception, e:
+ print(u'INSERT INTO admin1_code (country_id, geoname_id, code, name, ascii_name) VALUES (%s, %s, %s, %s, %s)' % (country_id, geoname_id, code, name, ascii_name))
+ if 'Duplicate' in str(e):
+ #print "I think the data was already entered."
+ return True
+ print "Got an error %s" % (e)
+ self.handle_exception(e, line)
+
+ self.admin1_codes.setdefault(country_id, {})
+ self.admin1_codes[country_id][code] = self.last_row_id('admin1_code', 'id')
+ line = fd.readline()[:-1]
+ fd.close()
+ print '%d first level administrative divisions imported' % self.table_count('admin1_code')
+
+ def import_second_level_adm(self):
+ print 'Importing second level administrative divisions'
+ fd = open('admin2Codes.txt')
+ line = fd.readline()[:-1]
+ while line:
+ codes, name, ascii_name, geoname_id = line.split('\t')
+ country_id, adm1, code = codes.split('.', 2)
+ try:
+ name = unicode(name,'utf-8')
+ except Exception, inst:
+ raise Exception("Encountered an error trying to encode the value for this line:\n%s\n\nThe error was: %s" (line, inst))
+ try:
+ admin1 = self.admin1_codes[country_id][adm1]
+ except KeyError:
+ admin1 = None
+ try:
+ self.cursor.execute(u'INSERT INTO admin2_code (country_id, admin1_id, geoname_id, code, name, ascii_name) VALUES (%s, %s, %s, %s, %s, %s)', (country_id, admin1, geoname_id, code, name, ascii_name))
+ except Exception, e:
+ if 'Duplicate' in str(e):
+ #print "I think the data was already entered."
+ return True
+ self.handle_exception(e, line)
+
+ self.admin2_codes.setdefault(country_id, {})
+ self.admin2_codes[country_id].setdefault(adm1, {})
+ self.admin2_codes[country_id][adm1][code] = self.last_row_id('admin2_code', 'id')
+ line = fd.readline()[:-1]
+ fd.close()
+ print '%d second level administrative divisions imported' % self.table_count('admin2_code')
+
+ def import_third_level_adm(self):
+ print 'Importing third level administrative divisions'
+ fd = open('allCountries.txt')
+ line = fd.readline()[:-1]
+ while line:
+ fields = line.split('\t')
+ fcode = fields[7]
+ if fcode != 'ADM3':
+ line = fd.readline()[:-1]
+ continue
+ geoname_id = fields[0]
+ name = fields[1]
+ try:
+ name = unicode(name,'utf-8')
+ except Exception, inst:
+ raise Exception("Encountered an error trying to encode the value for this line:\n%s\n\nThe error was: %s" (line, inst))
+ ascii_name = fields[2]
+ country_id = fields[8]
+ admin1 = fields[10]
+ admin2 = fields[11]
+ admin3 = fields[12]
+ admin1_id, admin2_id = [None] * 2
+ if admin1:
+ try:
+ admin1_id = self.admin1_codes[country_id][admin1]
+ except KeyError:
+ pass
+ if admin2:
+ try:
+ admin2_id = self.admin2_codes[country_id][admin1][admin2]
+ except KeyError:
+ pass
+ try:
+ self.cursor.execute(u'INSERT INTO admin3_code (country_id, admin1_id, admin2_id, geoname_id, code, name, ascii_name) VALUES (%s, %s, %s, %s, %s, %s, %s)', (country_id, admin1_id, admin2_id, geoname_id, admin3, name, ascii_name))
+ except Exception, e:
+ if 'Duplicate' in str(e):
+ #print "I think the data was already entered."
+ return True
+ self.handle_exception(e, line)
+
+ self.admin3_codes.setdefault(country_id, {})
+ self.admin3_codes[country_id].setdefault(admin1, {})
+ self.admin3_codes[country_id][admin1].setdefault(admin2, {})
+ self.admin3_codes[country_id][admin1][admin2][admin3] = self.last_row_id('admin3_code', 'id')
+
+ line = fd.readline()[:-1]
+
+ fd.close()
+ print '%d third level administrative divisions imported' % self.table_count('admin3_code')
+
+ def import_fourth_level_adm(self):
+ print 'Importing fourth level administrative divisions'
+ fd = open('allCountries.txt')
+ line = fd.readline()[:-1]
+ while line:
+ fields = line.split('\t')
+ fcode = fields[7]
+ if fcode != 'ADM4':
+ line = fd.readline()[:-1]
+ continue
+ geoname_id = fields[0]
+ name = fields[1]
+ try:
+ name = unicode(name,'utf-8')
+ except Exception, inst:
+ raise Exception("Encountered an error trying to encode the value for this line:\n%s\n\nThe error was: %s" (line, inst))
+ ascii_name = fields[2]
+ country_id = fields[8]
+ admin1 = fields[10]
+ admin2 = fields[11]
+ admin3 = fields[12]
+ admin4 = fields[13]
+ admin1_id, admin2_id, admin3_id = [None] * 3
+ if admin1:
+ try:
+ admin1_id = self.admin1_codes[country_id][admin1]
+ except KeyError:
+ pass
+ if admin2:
+ try:
+ admin2_id = self.admin2_codes[country_id][admin1][admin2]
+ except KeyError:
+ pass
+ if admin3:
+ try:
+ admin3_id = self.admin3_codes[country_id][admin1][admin2][admin3]
+ except KeyError:
+ pass
+ try:
+ self.cursor.execute(u'INSERT INTO admin4_code (country_id, admin1_id, admin2_id, admin3_id, geoname_id, code, name, ascii_name) VALUES (%s, %s, %s, %s, %s, %s, %s, %s)', (country_id, admin1_id, admin2_id, admin3_id, geoname_id, admin4, name, ascii_name))
+ except Exception, e:
+ if 'Duplicate' in str(e):
+ #print "I think the data was already entered."
+ return True
+ self.handle_exception(e, line)
+
+ self.admin4_codes.setdefault(country_id, {})
+ self.admin4_codes[country_id].setdefault(admin1, {})
+ self.admin4_codes[country_id][admin1].setdefault(admin2, {})
+ self.admin4_codes[country_id][admin1][admin2].setdefault(admin3, {})
+ self.admin4_codes[country_id][admin1][admin2][admin3][admin4] = self.last_row_id('admin4_code', 'id')
+
+ line = fd.readline()[:-1]
+
+ fd.close()
+ print '%d fourth level administrative divisions imported' % self.table_count('admin4_code')
+
+
+ def import_geonames(self):
+ print 'Importing geonames (this is going to take a while)'
+ fd = open('allCountries.txt')
+ line = fd.readline()[:-1]
+ while line:
+ fields = line.split('\t')
+ id, name, ascii_name = fields[:3]
+ latitude, longitude, fclass, fcode, country_id, cc2 = fields[4:10]
+ population, elevation, gtopo30 = fields[14:17]
+ if fclass != 'P': #only import populated places!
+ line = fd.readline()[:-1]
+ continue
+ moddate = fields[18]
+ if elevation == '':
+ elevation = 0
+ try:
+ timezone_id = self.time_zones[fields[17]]
+ except KeyError:
+ timezone_id = None
+ #XXX
+ try:
+ name = unicode(name,'utf-8')
+ except Exception, inst:
+ raise Exception("Encountered an error trying to encode the value for this line:\n%s\n\nThe error was: %s" (line, inst))
+ admin1 = fields[10]
+ admin2 = fields[11]
+ admin3 = fields[12]
+ admin4 = fields[13]
+ admin1_id, admin2_id, admin3_id, admin4_id = [None] * 4
+
+ if admin1:
+ try:
+ admin1_id = self.admin1_codes[country_id][admin1]
+ except KeyError:
+ pass
+
+ if admin2:
+ try:
+ admin2_id = self.admin2_codes[country_id][admin1][admin2]
+ except KeyError:
+ pass
+
+ if admin3:
+ try:
+ admin3_id = self.admin3_codes[country_id][admin1][admin2][admin3]
+ except KeyError:
+ pass
+
+ if admin4:
+ try:
+ admin4_id = self.admin4_codes[country_id][admin1][admin2][admin3][admin4]
+ except KeyError:
+ pass
+ try:
+ self.cursor.execute(u'INSERT INTO geoname (id, name, ascii_name, latitude, longitude, fclass, fcode, country_id, cc2, admin1_id, admin2_id, admin3_id, admin4_id, population, elevation, gtopo30, timezone_id, moddate) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)', (id, name, ascii_name, latitude, longitude, fclass, fcode, country_id, cc2, admin1_id, admin2_id, admin3_id, admin4_id, population, elevation, gtopo30, timezone_id, moddate))
+ except Exception, e:
+ if 'Duplicate' in str(e):
+ #print "I think the data was already entered."
+ return True
+ self.handle_exception(e, line)
+
+ line = fd.readline()[:-1]
+ fd.close()
+
+ print '%d geonames imported' % self.table_count('geoname')
+
+ def import_all(self):
+ self.pre_import()
+ self.begin()
+ self.import_fcodes()
+ self.commit()
+ self.begin()
+ self.import_language_codes()
+ self.commit()
+ self.begin()
+ self.import_alternate_names()
+ self.commit()
+ self.begin()
+ self.import_time_zones()
+ self.commit()
+ self.begin()
+ self.import_continent_codes()
+ self.commit()
+ self.begin()
+ self.import_countries()
+ self.commit()
+ self.begin()
+ self.import_first_level_adm()
+ self.commit()
+ self.begin()
+ self.import_second_level_adm()
+ self.commit()
+ self.begin()
+ self.import_third_level_adm()
+ self.commit()
+ self.begin()
+ self.import_fourth_level_adm()
+ self.commit()
+ self.begin()
+ self.import_geonames()
+ self.commit()
+ self.post_import()
+
+class PsycoPg2Importer(GeonamesImporter):
+ def table_count(self, table):
+ return 0
+
+ def pre_import(self):
+ self.end_stmts = []
+ import re
+ from django.core.management.color import no_style
+ from django.core.management.sql import sql_all
+ from django.db import models
+ sys.path.append('../')
+ sys.path.append('../../')
+
+ alter_re = re.compile('^ALTER TABLE "(\w+)" ADD CONSTRAINT (\w+).*', re.I)
+ alter_action = 'ALTER TABLE "\g<1>" DROP CONSTRAINT "\g<2>"'
+ index_re = re.compile('^CREATE INDEX "(\w+)".*', re.I)
+ index_action = 'DROP INDEX "\g<1>"'
+ table_re = re.compile('^CREATE TABLE "(\w+)".*', re.I)
+ references_re = re.compile('"(\w+)".*?REFERENCES "(\w+)" \("(\w+)"\) DEFERRABLE INITIALLY DEFERRED')
+ references_action = 'ALTER TABLE "%(table)s" DROP CONSTRAINT "%(table)s_%(field)s_fkey"'
+ references_stmt = 'ALTER TABLE "%(table)s" ADD CONSTRAINT "%(table)s_%(field)s_fkey" FOREIGN KEY ("%(field)s") ' \
+ 'REFERENCES "%(reftable)s" ("%(reffield)s") DEFERRABLE INITIALLY DEFERRED'
+ sql = sql_all(models.get_app('geonames'), no_style(), connections[DEFAULT_DB_ALIAS])
+ for stmt in sql:
+ if alter_re.search(stmt):
+ self.cursor.execute(alter_re.sub(alter_action, stmt))
+ self.end_stmts.append(stmt)
+ elif index_re.search(stmt):
+ self.cursor.execute(index_re.sub(index_action, stmt))
+ self.end_stmts.append(stmt)
+ elif table_re.search(stmt):
+ table = table_re.search(stmt).group(1)
+ for m in references_re.findall(stmt):
+ self.cursor.execute(references_action % \
+ {
+ 'table': table,
+ 'field': m[0],
+ 'reftable': m[1],
+ 'reffield': m[2],
+ })
+ self.end_stmts.append(references_stmt % \
+ {
+ 'table': table,
+ 'field': m[0],
+ 'reftable': m[1],
+ 'reffield': m[2],
+ })
+
+ self.cursor.execute('COMMIT')
+
+ def post_import(self):
+ print 'Enabling constraings and generating indexes (be patient, this is the last step)'
+ self.insert_dummy_records()
+ for stmt in self.end_stmts:
+ self.cursor.execute(stmt)
+ self.commit()
+
+ def insert_dummy_records(self):
+ self.cursor.execute("UPDATE geoname SET country_id='' WHERE country_id IN (' ', ' ')")
+ self.cursor.execute("DELETE FROM country WHERE geoname_id=6295630 or iso_numeric=-1")
+ self.cursor.execute("DELETE FROM continent WHERE geoname_id=6295630")
+ self.cursor.execute("INSERT INTO country (iso_alpha2, iso_alpha3, iso_numeric, fips_code, name, capital, area, population, continent_id, tld, currency_code, currency_name, phone_prefix, postal_code_fmt, postal_code_re, languages, geoname_id) VALUES ('', '', -1, '', 'No country', 'No capital', 0, 0, '', '', '', '', '', '', '', '', 6295630)")
+ self.cursor.execute("INSERT INTO continent VALUES('', 'No continent', 6295630)")
+
+ def begin(self):
+ self.cursor.execute('BEGIN')
+
+ def commit(self):
+ self.cursor.execute('COMMIT')
+
+ def get_db_conn(self):
+ import psycopg2
+ conn_params = 'dbname=%s ' % self.db
+ if self.host:
+ conn_params += 'host=%s ' % self.host
+ if self.user:
+ conn_params += 'user=%s ' % self.user
+ if self.password:
+ conn_params += 'password=%s' % self.password
+
+ self.conn = psycopg2.connect(conn_params)
+ self.cursor = self.conn.cursor()
+
+ def last_row_id(self, table=None, pk=None):
+ self.cursor.execute("SELECT CURRVAL('\"%s_%s_seq\"')" % (table, pk))
+ return self.cursor.fetchone()[0]
+
+ def set_import_date(self):
+ self.cursor.execute('INSERT INTO geonames_update (updated_date) VALUES ( CURRENT_DATE AT TIME ZONE \'UTC\')')
+
+class MySQLImporter(GeonamesImporter):
+ def table_count(self, table):
+ return 0
+
+ def import_file(self, tablename, filename):
+ import re
+ if re.search(r'[^\w\.]',tablename):
+ raise Exception("Illegal tablename: %s" % tablename)
+ try:
+ open(filename)
+ except:
+ raise Exception("Bad file.")
+ fullpath = "%s/%s" % (os.getcwd(),filename)
+ print "LOAD DATA INFILE '%(filename)s' IGNORE INTO TABLE `%(tablename)s` CHARACTER SET utf8" % {'tablename':tablename, 'filename':fullpath}
+ self.cursor.execute("LOAD DATA INFILE '%(filename)s' IGNORE INTO TABLE `%(tablename)s`" % {'tablename':tablename, 'filename':fullpath})
+
+ def pre_import(self):
+ self.end_stmts = []
+ import re
+ from django.core.management.color import no_style
+ from django.core.management.sql import sql_all
+ from django.db import models
+ sys.path.append('../')
+ sys.path.append('../../')
+
+ alter_re = re.compile('^ALTER TABLE "(\w+)" ADD CONSTRAINT (\w+).*', re.I)
+ alter_action = 'ALTER TABLE "\g<1>" DROP CONSTRAINT "\g<2>"'
+ index_re = re.compile('^CREATE INDEX "(\w+)".*', re.I)
+ index_action = 'DROP INDEX "\g<1>"'
+ table_re = re.compile('^CREATE TABLE "(\w+)".*', re.I)
+ references_re = re.compile('"(\w+)".*?REFERENCES "(\w+)" \("(\w+)"\) DEFERRABLE INITIALLY DEFERRED')
+ references_action = 'ALTER TABLE "%(table)s" DROP CONSTRAINT "%(table)s_%(field)s_fkey"'
+ references_stmt = 'ALTER TABLE "%(table)s" ADD CONSTRAINT "%(table)s_%(field)s_fkey" FOREIGN KEY ("%(field)s") ' \
+ 'REFERENCES "%(reftable)s" ("%(reffield)s")'
+ sql = sql_all(models.get_app('geonames'), no_style(), connections[DEFAULT_DB_ALIAS])
+ for stmt in sql:
+ if alter_re.search(stmt):
+ self.cursor.execute(alter_re.sub(alter_action, stmt))
+ self.end_stmts.append(stmt)
+ elif index_re.search(stmt):
+ self.cursor.execute(index_re.sub(index_action, stmt))
+ self.end_stmts.append(stmt)
+ elif table_re.search(stmt):
+ table = table_re.search(stmt).group(1)
+ for m in references_re.findall(stmt):
+ self.cursor.execute(references_action % \
+ {
+ 'table': table,
+ 'field': m[0],
+ 'reftable': m[1],
+ 'reffield': m[2],
+ })
+ self.end_stmts.append(references_stmt % \
+ {
+ 'table': table,
+ 'field': m[0],
+ 'reftable': m[1],
+ 'reffield': m[2],
+ })
+
+ self.cursor.execute('COMMIT')
+
+ def post_import(self):
+ print 'Enabling constraings and generating indexes (be patient, this is the last step)'
+ self.insert_dummy_records()
+ for stmt in self.end_stmts:
+ self.cursor.execute(stmt)
+ self.commit()
+
+ def insert_dummy_records(self):
+ self.cursor.execute("DELETE FROM country WHERE geoname_id=6295630 or iso_numeric=-1")
+ self.cursor.execute("DELETE FROM continent WHERE geoname_id=6295630")
+ self.cursor.execute("UPDATE geoname SET country_id='' WHERE country_id IN (' ', ' ')")
+ self.cursor.execute("INSERT INTO country VALUES ('', '', -1, '', 'No country', 'No capital', 0, 0, '', '', '', '', '', '', '', '', 6295630)")
+ self.cursor.execute("INSERT INTO continent VALUES('', 'No continent', 6295630)")
+
+ def begin(self):
+ self.cursor.execute('BEGIN')
+
+ def commit(self):
+ self.cursor.execute('COMMIT')
+
+ def get_db_conn(self):
+ import MySQLdb
+ conn_params = {}
+ conn_params['db'] = self.db
+ if self.host:
+ conn_params['host'] = self.host
+ if self.user:
+ conn_params['user'] = self.user
+ if self.password:
+ conn_params['passwd'] = self.password
+
+ conn_params['use_unicode'] = True
+
+ self.conn = MySQLdb.connect(**conn_params)
+ self.conn.set_character_set('utf8')
+ self.cursor = self.conn.cursor()
+ self.cursor.execute('SET NAMES utf8;')
+ self.cursor.execute('SET CHARACTER SET utf8;')
+ self.