Permalink
Browse files

Fixes by Ariel Mauricio Nunez Gomez

Remove custom SQL, except for translation
Add geometry fields to administrative divisions
Convert geonames-import into a management command
  • Loading branch information...
1 parent a1b0aee commit a479c5d903924af24dbd35b1caeab66a2d93fdd3 @fiam fiam committed Feb 16, 2009
View
@@ -0,0 +1,68 @@
+# -*- coding: utf-8 -*-
+
+# The contents of this file are subject to the Common Public Attribution License
+# Version 1.0 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.cpal-dev.com/cpaldemoopen/license. The License is based on the
+# Mozilla Public License Version 1.1 but Sections 14 and 15 have been added to cover
+# use of software over a computer network and provide for limited attribution for
+# the Original Developer. In addition, Exhibit A has been modified to be consistent
+# with Exhibit B.
+
+# Software distributed under the License is distributed on an "AS IS" basis, WITHOUT
+# WARRANTY OF ANY KIND, either express or implied. See the License for the specific
+# anguage governing rights and limitations under the License.
+
+# The Original Code is the byNotes project
+
+# The Initial Developer of the Original Code is Alberto García Hierro
+# The Original Developer is Alberto García Hierro
+
+# All portions of the code written by Alberto García Hierro are
+# Copyright (C) 2008 Alberto García Hierro
+# All Rights Reserved.
+
+from django.core.cache import cache
+
+def cache_set(key, value):
+ cache.set(key, value)
+ return value
+
+def _cached(func):
+ def cached_func(self):
+ key = 'cached_property_%s_%s_%s' % \
+ (self.__class__.__name__, func.__name__, self.pk)
+ val = cache.get(key)
+ return cache_set(key, func(self)) if val is None else val
+
+ cached_func.__name__ = func.__name__
+ return cached_func
+
+def cached_property(func):
+ return property(_cached(func))
+
+def _stored(func):
+ key = '_cached_%s' % func.__name__
+ def stored_func(self):
+ if not hasattr(self, key):
+ setattr(self, key, func(self))
+ return getattr(self, key)
+
+ stored_func.__name__ = func.__name__
+ return stored_func
+
+def stored_property(func):
+ return property(_stored(func))
+
+def full_cached_property(func):
+ return stored_property(_cached(func))
+
+def cached_method(func):
+ def cached_func(self, *args, **kwargs):
+ key = 'cached_method_%s_%s_%s_%s_%s' % \
+ (self.__class__.__name__, func.__name__, self.pk, hash(args),
+ hash(kwargs.iteritems()))
+ val = cache.get(key)
+ return cache_set(key, func(self, *args, **kwargs)) if val is None else val
+
+ return cached_func
View
@@ -1,196 +0,0 @@
-#!/usr/bin/python
-
-# This file is part of Django-Geonames
-# Copyright (c) 2008, Alberto Garcia Hierro
-# See LICENSE file for details
-
-UPDATE_SERVER_URI = 'http://abra.rm-fr.net/~fiam/geonames/'
-
-import os
-import sys
-import urllib2
-from warnings import filterwarnings
-from optparse import OptionParser
-from datetime import date, timedelta
-
-def get_file_fd(base_name, update_date):
- uri = '%(base_uri)s/%(base_name)s/%(base_name)s-%(date)s.txt' % \
- { 'base_uri': UPDATE_SERVER_URI,
- 'base_name': base_name,
- 'date': update_date.isoformat()
- }
-
- try:
- fd = urllib2.urlopen(uri)
- return fd
- except urllib2.HTTPError:
- print 'Cannot fetch updates for %s at %s. Exiting.' % (base_name, update_date)
- sys.exit(1)
-
-def apply_deletion(fd, klass):
- for line in fd.readlines():
- line = line[:-1]
- fields = line.split('\t')
- pk = fields[0]
- try:
- obj = klass.objects.get(pk=pk)
- obj.delete()
- except klass.DoesNotExist:
- pass
- fd.close()
-
-def apply_geonames_modifications(fd):
- from geonames.models import Geoname, Country, Admin1Code, Admin2Code, Admin3Code, Admin4Code, Timezone
- for line in fd.readlines():
- line = line[:-1]
- fields = line.split('\t')
- try:
- obj = Geoname.objects.get(pk=fields[0])
- except Geoname.DoesNotExist:
- obj = Geoname(pk=fields[0])
- obj.name = fields[1]
- obj.ascii_name = fields[2]
- obj.latitude = float(fields[4])
- obj.longitude = float(fields[5])
- obj.fclass = fields[6]
- obj.fcode = fields[7]
- obj.country = Country.objects.get(pk=fields[8])
- obj.cc2 = fields[9]
-
- if obj.fcode == 'ADM1':
- try:
- adm = Admin1Code.objects.get(geoname=obj)
- except Admin1Code.DoesNotExist:
- adm = Admin1Code(geoname=obj)
- adm.name = obj.name
- adm.ascii_name = obj.ascii_name
- adm.country = obj.country
- adm.code = fields[10]
- adm.save()
- if obj.fcode == 'ADM2':
- try:
- adm = Admin2.objects.get(geoname=obj)
- except Admin2Code.DoesNotExist:
- adm = Admin2Code(geoname=obj)
- adm.name = obj.name
- adm.ascii_name = obj.ascii_name
- adm.country = obj.country
- adm.code = fields[11]
- adm.save()
- if obj.fcode == 'ADM3':
- try:
- adm = Admin3Code.objects.get(geoname=obj)
- except Admin3Code.DoesNotExist:
- adm = Admin3Code(geoname=obj)
- adm.name = obj.name
- adm.ascii_name = obj.ascii_name
- adm.country = obj.country
- adm.code = fields[12]
- adm.save()
- if obj.fcode == 'ADM4':
- try:
- adm = Admin4Code.objects.get(geoname=obj)
- except Admin4Code.DoesNotExist:
- adm = Admin4Code(geoname=obj)
- adm.name = obj.name
- adm.ascii_name = obj.ascii_name
- adm.country = obj.country
- adm.code = fields[13]
- adm.save()
-
- obj.admin1, obj.admin2, obj.admin3, obj.admin4 = [None] * 4
- if fields[10]:
- try:
- obj.admin1 = Admin1Code.objects.get(country=obj.country, code=fields[10])
- except Admin1Code.DoesNotExist:
- pass
- if fields[11]:
- try:
- obj.admin2 = Admin2Code.objects.get(country=obj.country, admin1=obj.admin1, code=fields[11])
- except Admin2Code.DoesNotExist:
- pass
- if fields[12]:
- try:
- obj.admin3 = Admin3Code.objects.get(country=obj.country, admin1=obj.admin1, admin2=obj.admin2, code=fields[12])
- except Admin3Code.DoesNotExist:
- pass
- if fields[13]:
- try:
- obj.admin4 = Admin4Code.objects.get(country=obj.country, admin1=obj.admin1, admin2=obj.admin2, admin3=obj.admin3, code=fields[13])
- except Admin4Code.DoesNotExist:
- pass
- obj.population = int(fields[14] or 0)
- obj.elevation = int(fields[15] or 0)
- obj.gtopo30 = int(fields[16] or 0)
- try:
- obj.timezone = Timezone.objects.get(name=fields[17])
- except Timezone.DoesNotExist:
- obj.timezone = None
- obj.moddate = date(*[int(x) for x in fields[18].split('-')])
- obj.save()
-
-
- fd.close()
-
-def apply_altnames_modifications(fd):
- from geonames.models import Geoname, GeonameAlternateName
- for line in fd.readlines():
- line = line[:-1]
- fields = line.split('\t')
- try:
- obj = GeonameAlternateName.objects.get(pk=fields[0])
- except GeonameAlternateName.DoesNotExist:
- obj = GeonameAlternateName(pk=fields[0])
- obj.geoname = Geoname.objects.get(pk=fields[1])
- obj.language = fields[2]
- obj.name = fields[3]
- obj.preferred = int(fields[4] or 0)
- obj.short = int(fields[5] or 0)
- obj.save()
-
- fd.close()
-
-
-def main():
- parser = OptionParser()
-
- parser.add_option('-s', '--settings', action='store', type='string', dest='settings', default='settings')
-
- (options, args) = parser.parse_args(sys.argv)
-
- prg_name = sys.argv[0]
- if prg_name[0] == '.':
- prg_name = prg_name[1:]
- app_dir = os.path.dirname(os.getcwd() + '/' + prg_name)
- app_dir = app_dir.replace('//', '/')
- proj_dir = '/'.join(app_dir.split('/')[:-1])
- sys.path.append(proj_dir)
-
- proj_settings = __import__(options.settings)
- from django.core.management import setup_environ
- setup_environ(proj_settings)
- from django.conf import settings
- from geonames.models import Geoname, GeonameAlternateName, GeonamesUpdate
-
- filterwarnings(action='ignore', message='.*Field \'gpoint\' doesn\'t have a default value.*')
- try:
- updated_date = GeonamesUpdate.objects.latest('updated_date').updated_date
- except GeonamesUpdate.DoesNotExist:
- print 'Cannot find last update date'
- sys.exit(1)
-
- today = date.today()
- while updated_date != today:
- print 'Applying updates for ', updated_date
- for deletion, klass in [('deletes', Geoname), ('alternateNamesDeletes', GeonameAlternateName)]:
- fd = get_file_fd(deletion, updated_date)
- apply_deletion(fd, klass)
- fd = get_file_fd('modifications', updated_date)
- apply_geonames_modifications(fd)
- fd = get_file_fd('alternateNamesModifications', updated_date)
- apply_altnames_modifications(fd)
- updated_date += timedelta(days=1)
- GeonamesUpdate.objects.create(updated_date=updated_date)
-
-if __name__ == '__main__':
- main()
View
No changes.
No changes.
@@ -1,15 +1,11 @@
-#!/usr/bin/python
-
-# This file is part of Django-Geonames
-# Copyright (c) 2008, Alberto Garcia Hierro
-# See LICENSE file for details
-
import os
import sys
from optparse import OptionParser
from warnings import filterwarnings
from getpass import getpass
from datetime import date
+from django.conf import settings
+from django.core.management.base import BaseCommand
FILES = [
'http://download.geonames.org/export/dump/allCountries.zip',
@@ -59,13 +55,13 @@ def commit(self):
pass
def last_row_id(self, table=None, pk=None):
- raise NotImplementedError('This is a generic importer use one of the subclasses')
+ raise NotImplementedError('This is a generic importer, use one of the subclasses')
def get_db_conn(self):
- raise NotImplementedError('This is a generic importer use one of the subclasses')
+ raise NotImplementedError('This is a generic importer, use one of the subclasses')
def set_import_date(self):
- raise NotImplementedError('This is a generic importer use one of the subclasses')
+ raise NotImplementedError('This is a generic importer, use one of the subclasses')
def fetch(self):
try:
@@ -495,7 +491,6 @@ def post_import(self):
def insert_dummy_records(self):
self.cursor.execute("UPDATE geoname SET country_id='' WHERE country_id IN (' ', ' ')")
self.cursor.execute("INSERT INTO country VALUES ('', '', -1, '', 'No country', 'No capital', 0, 0, '', '', '', '', '', '', '', '', 6295630)")
- #self.cursor.execute("INSERT INTO country VALUES ('CS', '_CS', 0, '', 'To be split', 'No capital', 0, 0, '', '', '', 6295630)")
self.cursor.execute("INSERT INTO continent VALUES('', 'No continent', 6295630)")
def begin(self):
@@ -528,31 +523,8 @@ def set_import_date(self):
'postgresql_psycopg2': PsycoPg2Importer,
}
-def main():
-# filterwarnings('error')
-
- parser = OptionParser()
-
- parser.add_option('-s', '--settings', action='store', type='string',
- dest='settings', default='settings')
- parser.add_option('-t', '--tmpdir', action='store', type='string',
- dest='tmpdir', default='tmp')
-
- (options, args) = parser.parse_args(sys.argv)
-
- prg_name = sys.argv[0]
- if prg_name[0] == '.':
- prg_name = prg_name[1:]
- app_dir = os.path.dirname(os.getcwd() + '/' + prg_name)
- app_dir = app_dir.replace('//', '/')
- proj_dir = '/'.join(app_dir.split('/')[:-1])
- sys.path.append(proj_dir)
-
- proj_settings = __import__(options.settings)
- from django.core.management import setup_environ
- setup_environ(proj_settings)
- from django.conf import settings
-
+def main(options):
+ options = {'tmpdir':'geonames_temp'} #TODO: Make this a proper option
try:
importer = IMPORTERS[settings.DATABASE_ENGINE]
except KeyError:
@@ -564,13 +536,16 @@ def main():
user=settings.DATABASE_USER,
password=settings.DATABASE_PASSWORD,
db=settings.DATABASE_NAME,
- tmpdir=options.tmpdir)
+ tmpdir=options['tmpdir'])
imp.fetch()
imp.get_db_conn()
imp.import_all()
imp.set_import_date()
imp.cleanup()
-if __name__ == '__main__':
- main()
+class Command(BaseCommand):
+ help = "Geonames import command."
+
+ def handle(self, *args, **options):
+ main(options)
Oops, something went wrong.

0 comments on commit a479c5d

Please sign in to comment.