Save a list of search names to City

The geonames data has an handy comma separeted list of alternative
names for the city. Save it in db and hook the autocomplete plugins to
search there too. This let people search people in their native
language instead of in English.

cities_light/admin.py

@@ -30,7 +30,7 @@ class CityAdmin(admin.ModelAdmin):
     )
     search_fields = (
         'name',
-        'name_ascii',
+        'search_names',
     )
     list_filter = (
         'country__continent',

cities_light/autocomplete_light_registry.py

@@ -5,6 +5,8 @@
 autocomplete_light.register(Country)
 
 class CityChannel(autocomplete_light.ChannelBase):
+    search_field = 'search_names'
+
     def query_filter(self, results):
         results = super(CityChannel, self).query_filter(results)
 

cities_light/forms.py

@@ -10,4 +10,4 @@ class Meta:
 class CityForm(forms.ModelForm):
     class Meta:
         model = City
-        exclude = ('name_ascii', 'slug')
+        exclude = ('name_ascii', 'search_names', 'slug')

cities_light/lookups.py

@@ -28,7 +28,7 @@ class CityLookup(StandardLookupChannel):
     def get_query(self, q, request):
         return City.objects.filter(
             Q(name__icontains=q) |
-            Q(ascii_name__icontains=q)
+            Q(search_names__icontains=q)
         ).select_related('country').distinct()
 
     def get_result(self, obj):

cities_light/management/commands/cities_light.py

@@ -5,6 +5,7 @@
 import logging
 import zipfile
 import optparse
+import unicodedata
 
 from django.db import models
 from django.core.management.base import BaseCommand
@@ -153,13 +154,20 @@ def country_import(self, file_path):
             country.tld = items[9][1:] # strip the leading dot
             country.save()
 
+    def _normalize_search_names(self, search_names):
+        if isinstance(search_names, str):
+             search_names = force_unicode(search_names)
+
+        return unicodedata.normalize('NFKD', search_names).encode(
+             'ascii', 'ignore')
+
     def city_import(self, file_path):
         for items in self.parse(file_path):
             try:
                 city_items_pre_import.send(sender=self, items=items)
             except InvalidItems:
                 continue
-            
+
             kwargs = dict(name=items[1], country=self._get_country(items[8]))
 
             try:
@@ -175,6 +183,13 @@ def city_import(self, file_path):
                 city.longitude = items[5]
                 save = True
 
+            if not city.search_names:
+                # remove commas for names that are empty after normalization
+                search_names = ','.join([n.strip() for n in self._normalize_search_names(items[3]).split(',') if n.strip()])
+                if search_names:
+                    city.search_names = search_names
+                    save = True
+
             if not city.geoname_id: # city may have been added manually
                 city.geoname_id = items[0]
                 save = True

cities_light/migrations/0003_auto__add_field_city_search_names.py

@@ -0,0 +1,48 @@
+# encoding: utf-8
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+
+        # Adding field 'City.search_names'
+        db.add_column('cities_light_city', 'search_names', self.gf('django.db.models.fields.TextField')(default='', db_index=True), keep_default=False)
+
+
+    def backwards(self, orm):
+
+        # Deleting field 'City.search_names'
+        db.delete_column('cities_light_city', 'search_names')
+
+
+    models = {
+        'cities_light.city': {
+            'Meta': {'ordering': "['name']", 'unique_together': "(('country', 'name'),)", 'object_name': 'City'},
+            'country': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['cities_light.Country']"}),
+            'geoname_id': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'latitude': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '8', 'decimal_places': '5', 'blank': 'True'}),
+            'longitude': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '8', 'decimal_places': '5', 'blank': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '200', 'db_index': 'True'}),
+            'name_ascii': ('django.db.models.fields.CharField', [], {'max_length': '200', 'db_index': 'True'}),
+            'search_names': ('django.db.models.fields.TextField', [], {'default': "''", 'db_index': 'True'}),
+            'slug': ('autoslug.fields.AutoSlugField', [], {'unique_with': '()', 'max_length': '50', 'populate_from': 'None', 'db_index': 'True'})
+        },
+        'cities_light.country': {
+            'Meta': {'ordering': "['name']", 'object_name': 'Country'},
+            'code2': ('django.db.models.fields.CharField', [], {'max_length': '2', 'unique': 'True', 'null': 'True', 'blank': 'True'}),
+            'code3': ('django.db.models.fields.CharField', [], {'max_length': '3', 'unique': 'True', 'null': 'True', 'blank': 'True'}),
+            'continent': ('django.db.models.fields.CharField', [], {'max_length': '2', 'db_index': 'True'}),
+            'geoname_id': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '200'}),
+            'name_ascii': ('django.db.models.fields.CharField', [], {'max_length': '200', 'db_index': 'True'}),
+            'slug': ('autoslug.fields.AutoSlugField', [], {'unique_with': '()', 'max_length': '50', 'populate_from': 'None', 'db_index': 'True'}),
+            'tld': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '5', 'blank': 'True'})
+        }
+    }
+
+    complete_apps = ['cities_light']

cities_light/models.py

@@ -50,9 +50,9 @@ def __unicode__(self):
 
 class City(models.Model):
     name = models.CharField(max_length=200, db_index=True)
-    name_ascii = models.CharField(max_length=200, db_index=True)
     slug = autoslug.AutoSlugField(populate_from='name_ascii', 
         unique_with=('country__name',))
+    search_names = models.TextField(db_index=True, default='')
 
     latitude = models.DecimalField(max_digits=8, decimal_places=5,
         null=True, blank=True)