Merge branch 'bug-549012' into development

apps/search/clients.py

@@ -2,6 +2,37 @@
 
 from .sphinxapi import SphinxClient
 
+import re
+
+MARKUP_PATTERNS = (
+    (r'^!+',),
+    (r'^;:',),
+    (r'^#',),
+    (r'\n|\r',),
+    (r'\{maketoc\}',),
+    (r'\{ANAME.*?ANAME\}',),
+    (r'\{[a-zA-Z]+.*?\}',),
+    (r'\{.*?$',),
+    (r'__',),
+    (r'\'\'',),
+    (r'%{2,}',),
+    (r'\*|\^|;|/\}',),
+    (r'~/?np~',),
+    (r'~/?(h|t)c~',),
+    (r'\(spans.*?\)',),
+    (r'\}',),
+    (r'\(\(.*?\|(?P<name>.*?)\)\)', '\g<name>'),
+    (r'\(\((?P<name>.*?)\)\)', '\g<name>'),
+    (r'\(\(',),
+    (r'\)\)',),
+    (r'\[.+?\|(?P<name>.+?)\]', '\g<name>'),
+    (r'\[(?P<name>.+?)\]', '\g<name>'),
+    (r'/wiki_up.*? ',),
+    (r'&quot;',),
+    (r'^!! Issue.+!! Description',),
+    (r'\s+',),
+)
+
 
 class SearchClient(object):
     """
@@ -13,13 +44,55 @@ def __init__(self):
         self.sphinx.SetServer(settings.SPHINX_HOST, settings.SPHINX_PORT)
         self.sphinx.SetLimits(0, settings.SEARCH_MAX_RESULTS)
 
+        # initialize regexes for markup cleaning
+        self.truncate_pattern = re.compile(r'\s.*', re.MULTILINE)
+        self.compiled_patterns = []
+
+        if MARKUP_PATTERNS:
+            for pattern in MARKUP_PATTERNS:
+                p = [re.compile(pattern[0], re.MULTILINE)]
+                if len(pattern) > 1:
+                    p.append(pattern[1])
+                else:
+                    p.append(' ')
+
+                self.compiled_patterns.append(p)
+
     def query(self, query, filters): abstract
 
+    def excerpt(self, result, query):
+        """
+        Returns an excerpt for the passed-in string
+
+        Takes in a string
+        """
+        documents = [result]
+
+        # build excerpts that are longer and truncate
+        # see multiplier constant definition for details
+        raw_excerpt = self.sphinx.BuildExcerpts(documents, self.index, query,
+            {'limit': settings.SEARCH_SUMMARY_LENGTH
+                * settings.SEARCH_SUMMARY_LENGTH_MULTIPLIER})[0]
+
+        excerpt = raw_excerpt
+        for p in self.compiled_patterns:
+            excerpt = p[0].sub(p[1], excerpt)
+
+        # truncate long excerpts
+        if len(excerpt) > settings.SEARCH_SUMMARY_LENGTH:
+            excerpt = excerpt[:settings.SEARCH_SUMMARY_LENGTH] \
+                + self.truncate_pattern.sub('', excerpt[settings.SEARCH_SUMMARY_LENGTH:])
+            if excerpt[-1] != '.':
+                excerpt += '...'
+
+        return excerpt
+
 
 class ForumClient(SearchClient):
     """
     Search the forum
     """
+    index = 'forum_threads'
 
     def query(self, query, filters=None):
         """
@@ -55,6 +128,7 @@ class WikiClient(SearchClient):
     """
     Search the knowledge base
     """
+    index = 'wiki_pages'
 
     def query(self, query, filters=None):
         """
@@ -77,7 +151,7 @@ def query(self, query, filters=None):
                 sc.SetFilter(f['filter'], f['value'],
                              f.get('exclude', False))
 
-        result = sc.Query(query, 'wiki_pages')
+        result = sc.Query(query, self.index)
 
         if result:
             return result['matches']

apps/search/templates/results.html

@@ -22,22 +22,20 @@
 
 
         {% for doc in results %}
-        <div class="oneresult">
-            <a class="searchresultlink" href="{{ doc.get_url() }}">{{ doc.name }}</a>
+        <div class="result">
+            <a class="title" href="{{ doc.url }}">{{ doc.title }}</a>
             <p>
-                <a class="blacklink" href="{{ doc.get_url() }}">
-                    {{ doc.search_summary }}
-                    ...
+                <a href="{{ doc.url }}">
+                    {{ doc.search_summary|safe }}
                 </a>
             </p>
         </div>
-        <div class="oneresulthr"></div>
         {% endfor %}
 
 
         {{ pages|paginator }}
 
     </div>
 
-        <p>Can't find what you're looking for? <a href="/kb/Ask+a+question">Ask a support question instead!</a></p>
+        <p>Can't find what you're looking for? <a href="/{{ locale }}/kb/Ask+a+question">Ask a support question instead!</a></p>
 {% endblock %}

apps/search/views.py

@@ -96,9 +96,19 @@ def search(request):
     for i in range(offset, offset + settings.SEARCH_RESULTS_PER_PAGE):
         try:
             if documents[i]['attrs'].get('category', False):
-                results.append(WikiPage.objects.get(pk=documents[i]['id']))
+                wiki_page = WikiPage.objects.get(pk=documents[i]['id'])
+                result = {'search_summary': wc.excerpt(wiki_page.data, q),
+                    'url': wiki_page.get_url(),
+                    'title': wiki_page.name,
+                    }
+                results.append(result)
             else:
-                results.append(ForumThread.objects.get(pk=documents[i]['id']))
+                forum_thread = ForumThread.objects.get(pk=documents[i]['id'])
+                result = {'search_summary': fc.excerpt(forum_thread.data, q),
+                    'url': forum_thread.get_url(),
+                    'title': forum_thread.name,
+                    }
+                results.append(result)
         except IndexError:
             break
         except (WikiPage.DoesNotExist, ForumThread.DoesNotExist):

apps/sumo/models.py

@@ -101,10 +101,6 @@ def __unicode__(self):
     def name(self):
         return self.pageName
 
-    @property
-    def search_summary(self):
-        return self.description
-
     def get_url(self):
         """
         TODO: Once we can use reverse(), use reverse()

media/css/search.css

@@ -2,6 +2,19 @@
     font-size: 110%;
 }
 
+.search-results .title {
+    font-size: 130%;
+}
+
+.search-results p a {
+    color: #000;
+    line-height: 140%;
+}
+
+.search-results .result {
+    margin-bottom: 15px;
+}
+
 .search-query {
     border: 1px solid #E16601;
     width: 270px;

settings.py

@@ -125,3 +125,7 @@
 # Search default settings
 SEARCH_DEFAULT_CATEGORIES = '1,17,18' # comma-separated string of category IDs
 SEARCH_DEFAULT_FORUM = '1' # default forum ID (eg: 1 on sumo, 5 on mosumo)
+SEARCH_SUMMARY_LENGTH = 275
+# because of markup cleanup, search summaries lengths vary quite a bit
+# so we extract longer excerpts and perform truncation to the length above
+SEARCH_SUMMARY_LENGTH_MULTIPLIER = 1.3