Merge pull request #57 from grapesmoker/unicode_searches

Unicode searches

collab/settings.py

@@ -222,4 +222,19 @@
 
 # If using elasticsearch, override search_analyzer for ngram/edgengram fields.
 # Otherwise, searching for "sample" will return any results that start with "sam"
-ELASTICSEARCH_DEFAULT_NGRAM_SEARCH_ANALYZER = 'standard'
+ELASTICSEARCH_INDEX_SETTINGS = {
+    'settings': {
+        'analysis': {
+            'analyzer': {
+                'default': {
+                    'type': 'custom',
+                    'tokenizer': 'standard',
+                    'filter': ['lowercase', 'asciifolding']
+                    },
+                }
+            }
+        }
+    }
+
+
+ELASTICSEARCH_DEFAULT_NGRAM_SEARCH_ANALYZER = 'default'

core/search/admin.py

@@ -1,4 +1,6 @@
 from django.contrib import admin
 from core.search.models import SearchableTool
+from core.search.models import SuggestedSearchResult
 
 admin.site.register(SearchableTool)
+admin.site.register(SuggestedSearchResult)

core/search/migrations/0002_add_model_SuggestedSearchResult.py

@@ -0,0 +1,43 @@
+# -*- coding: utf-8 -*-
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        # Adding model 'SuggestedSearchResult'
+        db.create_table(u'search_suggestedsearchresult', (
+            (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('search_term', self.gf('django.db.models.fields.CharField')(max_length=255)),
+            ('suggested_url', self.gf('django.db.models.fields.URLField')(max_length=200)),
+            ('description', self.gf('django.db.models.fields.TextField')()),
+        ))
+        db.send_create_signal(u'search', ['SuggestedSearchResult'])
+
+
+    def backwards(self, orm):
+        # Deleting model 'SuggestedSearchResult'
+        db.delete_table(u'search_suggestedsearchresult')
+
+
+    models = {
+        u'search.searchabletool': {
+            'Meta': {'object_name': 'SearchableTool'},
+            'date_added': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'link': ('django.db.models.fields.CharField', [], {'max_length': '2048'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'})
+        },
+        u'search.suggestedsearchresult': {
+            'Meta': {'object_name': 'SuggestedSearchResult'},
+            'description': ('django.db.models.fields.TextField', [], {}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'search_term': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            'suggested_url': ('django.db.models.fields.URLField', [], {'max_length': '200'})
+        }
+    }
+
+    complete_apps = ['search']

core/search/models.py

@@ -8,3 +8,22 @@ class SearchableTool(models.Model):
 
     def __unicode__(self):
         return u"%s" % self.name
+
+
+class SuggestedSearchResult(models.Model):
+
+    search_term = models.CharField(max_length=255)
+    suggested_url = models.URLField()
+    description = models.TextField()
+
+    def __unicode__(self):
+        return u"%s" % self.search_term
+
+    def to_dict(self):
+        return {'search_term': self.search_term,
+                'suggested_url': self.suggested_url,
+                'name': self.suggested_url
+                        .replace('https://team.cfpb.local/wiki/index.php/', '')
+                        .replace('_', ' ')
+                        .replace("#", ": "),
+                'description': self.description}

core/search/templates/search/search_results.html

@@ -10,6 +10,17 @@ <h1>Search results for '{{ term }}'</h1>
             {% if not index %}
             <div class="results-group" data-model="Wiki" style="display:none;">
                 <h3>Wiki (<span id="wiki_count_hook"></span> results)</h3>
+
+                {% if suggested_results %}
+                <div class="suggested-result">
+                    <h4>Suggested result</h4>
+                    <a href="{{ suggested_results.0.suggested_url }}">
+                       {{ suggested_results.0.name }}
+                    </a>
+                    <p>{{ suggested_results.0.description }}</p>
+                </div>
+                {% endif %}
+
                 <ol id="wiki_results_hook"></ol>
             </div>
             {% endif %}
@@ -25,22 +36,31 @@ <h3>There are no results that match your search.</h3>
 
                 {% for results_by_index in by_index %}
                 <div class="results-group" data-model="{{ results_by_index.grouper }}">
-                    <h3>{{ results_by_index.grouper }} ({{ results_by_index.list|length }} results)</h3>
+                    <h3>
+                        {{ results_by_index.grouper }}
+                    {% if results_by_index.list|length > 50 and results_by_index.grouper != 'Staff Directory' %}
+                        (showing first 50 results of {{ results_by_index.list|length }})
+                    {% else %}
+                        ({{ results_by_index.list|length }} results)
+                    {% endif %}
+                    </h3>
 
                     {% regroup results_by_index.list by content_type as by_content %}
                     {% for results_by_content in by_content %}
                         {% if results_by_content.grouper == 'core.person' %}
-                                <div class="profile_images">
+                                <ol class="profile_images cf">
                                     {% for p in results_by_content.list %}
-                                        <a href="{{ p.url }}">
-                                            <img class="tiny_photo" src="{{ p.image }}"/>
-                                            <span>{{ p.display }}</span>
-                                        </a>
+                                        <li>
+                                            <a href="{{ p.url }}">
+                                                <img class="tiny_photo" src="{{ p.image }}"/>
+                                                <span>{{ p.display }}</span>
+                                            </a>
+                                        </li>
                                     {% endfor %}
-                                </div>
+                                </ol>
                         {% else %}
-                            <ol>
-                                {% for item in results_by_content.list %}
+                            <ol data-results-by-content-grouper="{{ results_by_content.grouper }}">
+                               {% for item in results_by_content.list|slice:":50" %}
                                     <li>
                                         <a name="item_url" href="{{ item.url }}">
                                             {{ item.display }}

core/search/templates/search/search_results.js

@@ -9,7 +9,12 @@ function collapseResults( group ) {
     var results = group.find( 'ol' ).children( 'li' );
     if ( results.length > 5 && dataModel !== openGroup ) {
         results.hide().slice( 0, 5 ).show();
-        var link = $( '<a class="more_results btn" href="#">' + ( results.length - 5 ) + ' more results</a>' );
+        if (results.length > 50 && dataModel != 'Staff Directory') {
+            var numMore = 45;
+        } else {
+            var numMore = results.length - 5;
+        }
+        var link = $( '<a class="more_results btn" href="#">' + numMore + ' more results</a>' );
         link.click( function( e ) {
             e.preventDefault();
             results.slideDown();
@@ -22,7 +27,7 @@ function collapseResults( group ) {
 
 {% if wiki_installed %}
     {% autoescape off %}
-        $.getJSON( '{{ wiki_search_json_url }}', function( data ) {
+        $.getJSON( "{{ wiki_search_json_url }}", function( data ) {
             var results = data.query.search;
             var totalResults = data.query.searchinfo.totalhits;
             var dataModel = 'Wiki';
@@ -38,18 +43,8 @@ function collapseResults( group ) {
                     wikiCount.html( results.length );
                 }
                 wikiGroup.show();
-
-                var suggestedData = results[0];
-                var suggestedLink = suggestedData.title.replace( ' ', '_' );
-                var suggestedContent = suggestedData.snippet.replace( /<(\/)?div.*?>/gm, '' );
-                var suggestedHTML = '<div class="suggested-result">';
-                    suggestedHTML += '<h4>Suggested result</h4>';
-                    suggestedHTML += '<a href="/wiki/index.php/' + suggestedLink + '">';
-                    suggestedHTML += suggestedData.title + '</a>';
-                    suggestedHTML += '<p>' + suggestedContent + '</p></div>';
-                wikiList.prepend( suggestedHTML );
 
-                for ( var count = 1; count < results.length; count++ ) {
+                for ( var count = 0; count < results.length; count++ ) {
                     var resultData = results[count];
                     var resultLink = resultData.title.replace( ' ', '_' );
                     var resultContent = resultData.snippet.replace( /<(\/)?div.*?>/gm, '' );

core/search/views.py

@@ -1,5 +1,6 @@
 import re
 import itertools
+from collections import OrderedDict
 
 from django.shortcuts import render_to_response
 from django.http import HttpResponseRedirect
@@ -10,18 +11,32 @@
 from django.core.context_processors import csrf
 from django.views.decorators.csrf import csrf_response_exempt, csrf_exempt
 from django.conf import settings
+from django.utils.encoding import smart_text
 
 from haystack import connections
 from haystack.query import SearchQuerySet
 
 from core.utils import json_response
 from core.models import Person
 from core.taggit.models import Tag
-from core.search.models import SearchableTool
-
+from core.search.models import SuggestedSearchResult
 
 TEMPLATE_PATH = 'search/'
 
+# The search terms that are passed to Elasticsearch need to be escaped
+# to HTML entities because that's how Elastic looks them up. This table
+# is used to do that; it's ordered because you need to guarantee that
+# ampersands are replaced first in order not to break the string.
+html_escape_table = OrderedDict()
+html_escape_table['&'] = '&'
+html_escape_table["'"] = '''
+
+
+def escape(text, table):
+    result = text
+    for k, v in table.items():
+        result = result.replace(k, v)
+    return result
 
 def _get_indexes():
     index = connections['default'].get_unified_index()
@@ -56,6 +71,7 @@ def index(req, term=''):
 def search_results_json(req, term='', context_models=''):
     all_results = []
     term = req.GET.get('term', '')
+    escaped_term = escape(term, html_escape_table)
     context_models = req.GET.get('model', '').split(',')
 
     p = {}
@@ -64,7 +80,7 @@ def search_results_json(req, term='', context_models=''):
                      context_models else index[1].PRIORITY)
 
     for index in indexes:
-        results = SearchQuerySet().filter(content=term).models(index[0])
+        results = SearchQuerySet().filter(content=escaped_term).models(index[0])
         results_count = results.count()
         for r in results[:5]:
             all_results.append(_create_category(r.display,
@@ -73,7 +89,7 @@ def search_results_json(req, term='', context_models=''):
                                                 r.model_name,
                                                 r.url,
                                                 results_count))
-
+    
     return json_response(all_results)
 
 
@@ -90,31 +106,37 @@ def _create_category(label, category, term, search_slug, link, results_len):
 def search(req, term='', index=''):
     if term == '':
         if req.method == 'POST':
-            term = req.POST.get('term', '')
+            term = smart_text(req.POST.get('term', ''))
         else:
-            term = req.GET.get('term', '')
+            term = smart_text(req.GET.get('term', ''))
 
     term = term.strip()
     if term == '':
         return HttpResponseRedirect(reverse('search:index'))
+    escaped_term = escape(term, html_escape_table)
+
+    suggested_results = SuggestedSearchResult.objects.filter(search_term=term.lower())
 
     p = {}
     p['term'] = term
+    p['suggested_results'] = [res.to_dict() for res in suggested_results]
 
     if index != '' and index != 'all':
         p['index'] = index
         for i in _get_indexes():
             if i[0].__name__.lower() == index:
                 p['results'] = SearchQuerySet().filter(
-                    content=term).models(i[0]).order_by('index_priority', 'index_name', 'index_sort')
+                    content=escaped_term).models(i[0]).order_by('index_priority', 'index_name')
     else:
         p['results'] = SearchQuerySet().filter(
-            content=term).order_by('index_priority', 'index_name', 'index_sort')
+            content=escaped_term).order_by('index_priority', 'index_name')
 
     if settings.WIKI_INSTALLED:
         p['wiki_installed'] = True
+	escaped_term = "'" + escaped_term
+	escaped_term = escaped_term.replace('&', '%26')
         p['wiki_search_json_url'] = settings.WIKI_SEARCH_URL % \
-            ('50', term)
+            ('50', escaped_term)
     p['WIKI_URL_BASE'] = settings.WIKI_URL_BASE
 
     return render_to_response(TEMPLATE_PATH + 'search_results.html', p,