Merge pull request #807 from Natay/master

pagination added to search

biostar/forum/search.py

@@ -8,9 +8,9 @@
 from django.conf import settings
 from django.db.models import Q
 from whoosh import writing, classify
-from whoosh.analysis import StemmingAnalyzer
+from whoosh.analysis import StemmingAnalyzer, StopFilter
 from whoosh.writing import AsyncWriter, BufferedWriter
-from whoosh.searching import Results
+from whoosh.searching import Results, ResultsPage
 import html2markdown
 import bleach
 from whoosh.qparser import MultifieldParser, OrGroup
@@ -23,7 +23,10 @@
 logger = logging.getLogger('engine')
 
 # Stop words ignored where searching.
-STOP = ['there', 'where', 'who', 'that'] + [w for w in STOP_WORDS]
+STOP = ['there', 'where', 'who', 'that', 'to', 'do', 'my', 'only', 'but', 'about',
+        'our', 'able', 'how', 'am', 'so', 'want']
+
+STOP += [w for w in STOP_WORDS]
 STOP = set(STOP)
 
 
@@ -56,19 +59,20 @@ def copy_hits(result, highlight=False):
     """
     # Highlight title and content
     if highlight:
-        title = result.highlights("title", minscore=0)
-        title = title or result.get('title')
-        content = result.highlights("content", top=5, minscore=0)
-        content = content or result.get('content')
+        title = result.highlights("title", minscore=0) or result.get('title')
+        content = result.highlights("content", top=5, minscore=0) or result.get('content')
+        tags = result.highlights("tags", minscore=0) or result.get('tags')
     else:
         title = result.get('title')
         content = result.get('content')
+        tags = result.get('tags')
 
     bunched = dict(title=title,
                    content=content,
                    uid=result.get('uid'),
-                   tags=result.get('tags'),
-                   author=result.get('author'))
+                   tags=tags,
+                   author=result.get('author'),
+                   lastedit_date=result.get('lastedit_date'))
     return bunched
 
 
@@ -77,66 +81,26 @@ def index_exists(dirname=settings.INDEX_DIR, indexname=settings.INDEX_NAME):
 
 
 def add_index(post, writer):
-
     # Ensure the content is stripped of any html.
     content = bleach.clean(post.content, styles=[], attributes={}, tags=[], strip=True)
     writer.update_document(title=post.title,
                            content=content,
                            tags=post.tag_val,
                            author=post.author.profile.name,
-                           uid=post.uid)
+                           uid=post.uid,
+                           lastedit_date=post.lastedit_date)
 
 
 def get_schema():
-    analyzer = StemmingAnalyzer(stoplist=STOP)
+    analyzer = StemmingAnalyzer(stoplist=STOP) | StopFilter(stoplist=STOP)
     schema = Schema(title=TEXT(analyzer=analyzer, stored=True, sortable=True),
                     content=TEXT(analyzer=analyzer, stored=True, sortable=True),
                     tags=KEYWORD(commas=True, stored=True),
                     author=TEXT(stored=True),
-                    uid=ID(stored=True))
+                    uid=ID(unique=True, stored=True),
+                    lastedit_date=DATETIME(sortable=True, stored=True))
     return schema
 
-#
-# def get_schema():
-#    """
-#    This is the issue!!!!!!
-#    """
-#     analyzer = StemmingAnalyzer(stoplist=STOP)
-#     schema = Schema(title=TEXT(analyzer=analyzer, sortable=True),
-#                     url=ID(stored=True),
-#                     content_length=NUMERIC(sortable=True),
-#                     thread_votecount=NUMERIC(stored=True, sortable=True),
-#                     vote_count=NUMERIC(stored=True, sortable=True),
-#                     content=TEXT(stored=True, analyzer=analyzer, sortable=True),
-#                     tags=KEYWORD(stored=True, commas=True),
-#                     is_toplevel=BOOLEAN(stored=True),
-#                     author_is_moderator=BOOLEAN(stored=True),
-#                     lastedit_user_is_moderator=BOOLEAN(stored=True),
-#                     lastedit_user_is_suspended=BOOLEAN(stored=True),
-#                     author_is_suspended=BOOLEAN(stored=True),
-#                     lastedit_date=DATETIME(stored=True, sortable=True),
-#                     creation_date=DATETIME(stored=True, sortable=True),
-#                     rank=NUMERIC(stored=True, sortable=True),
-#                     author=TEXT(stored=True),
-#                     lastedit_user=TEXT(stored=True),
-#                     lastedit_user_email=TEXT(stored=True),
-#                     lastedit_user_score=NUMERIC(stored=True, sortable=True),
-#                     lastedit_user_uid=ID(stored=True),
-#                     lastedit_user_url=ID(stored=True),
-#                     author_score=NUMERIC(stored=True, sortable=True),
-#                     author_handle=TEXT(stored=True),
-#                     author_email=TEXT(stored=True),
-#                     author_uid=ID(stored=True),
-#                     author_url=ID(stored=True),
-#                     root_has_accepted=BOOLEAN(stored=True),
-#                     reply_count=NUMERIC(stored=True, sortable=True),
-#                     view_count=NUMERIC(stored=True, sortable=True),
-#                     answer_count=NUMERIC(stored=True, sortable=True),
-#                     uid=ID(stored=True),
-#                     type=NUMERIC(stored=True, sortable=True),
-#                     type_display=TEXT(stored=True))
-#     return schema
-
 
 def init_index(dirname=None, indexname=None, schema=None):
     # Initialize a new index or return an already existing one.
@@ -231,7 +195,7 @@ def crawl(reindex=False, overwrite=False, limit=1000):
     return
 
 
-def whoosh_search(query, limit=10, ix=None, fields=None, sortedby=[], **kwargs):
+def whoosh_search(query, limit=10, page=1, ix=None, fields=None, reverse=False, sortedby=[], **kwargs):
     """
     Query search index
     """
@@ -246,38 +210,30 @@ def whoosh_search(query, limit=10, ix=None, fields=None, sortedby=[], **kwargs):
 
     parser = MultifieldParser(fieldnames=fields, schema=ix.schema, group=orgroup).parse(query)
 
-    hits = searcher.search(parser,
-                           sortedby=sortedby,
-                           terms=True,
-                           limit=limit)
-
-    # Allow larger fragments
-    hits.fragmenter.maxchars = 100
-    hits.fragmenter.surround = 100
+    hits = searcher.search_page(parser,pagenum=page, pagelen=limit, reverse=reverse, sortedby=sortedby, **kwargs)
+    hits.results.fragmenter.maxchars = 100
+    hits.results.fragmenter.surround = 100
 
     return hits
 
 
-def perform_search(query, fields=None, sortedby=[], limit=None):
+def perform_search(query, page=1, fields=None, reverse=False, sortedby=[], limit=None):
     """
     Utility functions to search whoosh index, collect results and closes
     """
 
     limit = limit or settings.SEARCH_LIMIT
 
-    hits = whoosh_search(query=query,
-                         fields=fields,
-                         sortedby=sortedby,
-                         limit=limit)
+    indexed = whoosh_search(query=query, fields=fields, page=page, reverse=reverse, sortedby=sortedby, limit=limit)
 
     # Highlight the whoosh results.
     copier = lambda r: copy_hits(r, highlight=True)
 
-    final = list(map(copier, hits))
-    # Ensure searcher object gets closed.
-    hits.searcher.close()
+    final = list(map(copier, indexed))
 
-    return final
+    indexed.results.searcher.close()
+
+    return final, indexed
 
 
 def more_like_this(uid, top=0, sortedby=[]):
@@ -287,20 +243,16 @@ def more_like_this(uid, top=0, sortedby=[]):
 
     top = top or settings.SIMILAR_FEED_COUNT
     fields = ['uid']
-    results = whoosh_search(query=uid, sortedby=sortedby, fields=fields)
+    found = whoosh_search(query=uid, sortedby=sortedby, fields=fields)
 
-    if len(results):
-        results = results[0].more_like_this("content", top=top)
+    if len(found):
+        hits = found[0].more_like_this("content", top=top)
         # Copy hits to list and close searcher object.
-        final = list(map(copy_hits, results))
-        # Show unique posts
-        final = {h['uid']: h for h in final}
-        final = list(final.values())
+        final = list(map(copy_hits, hits))
     else:
         final = []
 
-    # Ensure searcher object gets closed.
-    results.searcher.close()
+    found.results.searcher.close()
 
     return final
 

biostar/forum/static/forum.js

@@ -217,6 +217,7 @@ function highligh_preview(form, text) {
 
 }
 
+
 $(document).ready(function () {
 
     $('#similar-feed').each(function () {
@@ -336,12 +337,12 @@ $(document).ready(function () {
         $('.hidden-answer').toggle()
     });
 
+    tags_dropdown();
+
     $('pre').addClass('language-bash');
     $('code').addClass('language-bash');
     Prism.highlightAll();
 
-    tags_dropdown();
-
 
 })
 ;

biostar/forum/templates/search/search_pages.html

@@ -0,0 +1,33 @@
+{% load humanize %}
+{% load forum_tags %}
+    <div class="ui page-bar segment">
+{% if results.pagenum != 1 %}
+    <a class="ui small basic button no-shadow"
+
+       href="{% url 'post_search' %}?query={{ query }}&page={{ previous_page }}&order={{ order }}">
+        <i class="ui angle  double left icon"> </i>
+    </a>
+{% else %}
+    <div class="ui small basic button no-shadow">
+        <i class="ui angle  double left icon"> </i>
+    </div>
+{% endif %}
+
+<span class="phone">{{ results.total|intcomma }}
+    result{{ results.total|pluralize }}
+    &bull;
+    Page </span> {{ results.pagenum }} of {{ results.pagecount }}
+
+{% if not results.is_last_page %}
+
+    <a class="ui small basic button no-shadow"
+       href="{% url 'post_search' %}?query={{ query }}&page={{ next_page }}&order={{ order }}">
+
+        <i class="ui angle  double right icon"></i>
+    </a>
+    {% else %}
+    <div class="ui small basic button no-shadow">
+        <i class="ui angle  double right icon"></i>
+    </div>
+{% endif %}
+    </div>

biostar/forum/templates/search/search_results.html

@@ -12,8 +12,21 @@
         {% search_bar %}
 
     {% endblock %}
+    {% search_pages indexed %}
+
+  <div class="ui simple compact dropdown item">
+      Order by: {{ order|capfirst }}
+
+    <i class="dropdown icon"></i>
+    <div class="menu">
+      <a class="item" href="{% url 'post_search' %}?query={{ query }}&order=relevance">
+          <i class="star icon"></i> Relevance</a>
+      <a class="item" href="{% url 'post_search' %}?query={{ query }}&order=date">
+          <i class="calendar icon"></i>Date</a>
+
+    </div>
+  </div>
 
-    <div class="ui message"><i class="search icon"></i>{{ total }} posts containing: <b>{{ query }}</b></div>
 
     <div class="ui divided items">
         {% for result in results %}
@@ -33,7 +46,7 @@
                             {% post_tags tags_str=result.tags spaced=False %}
                         </div>
                         <div class="right floated muted">
-                            {{ result.author|truncatechars:40 }}
+                             updated {{ result.lastedit_date|time_ago }} &bull; {{ result.author|truncatechars:40 }}
                         </div>
                     </div>
                 </div>
@@ -45,7 +58,7 @@
             </div>
         {% endfor %}
     </div>
-
+ {% search_pages indexed %}
 {% endblock %}
 
 

biostar/forum/templates/widgets/post_tags.html

@@ -3,7 +3,7 @@
     {% for tag in tags %}
         {% if tag %}
         <a class="ptag" href="{% url 'post_tags' tag %}">
-            {{ tag }}
+            {{ tag |safe}}
         </a>
         {% endif %}
     {% endfor %}

biostar/forum/templatetags/forum_tags.py

@@ -16,6 +16,7 @@
 from django.utils.safestring import mark_safe
 from django.utils.timezone import utc
 from taggit.models import Tag
+from re import IGNORECASE, compile, escape
 
 from biostar.accounts.models import Profile, Message
 from biostar.forum import const, auth
@@ -415,6 +416,17 @@ def custom_feed(objs, ftype='', title=''):
     return context
 
 
+@register.inclusion_tag(takes_context=True, filename='search/search_pages.html')
+def search_pages(context, results):
+    previous_page = results.pagenum - 1
+    next_page = results.pagenum + 1 if not results.is_last_page() else results.pagenum
+    request = context['request']
+    query = request.GET.get('query', '')
+    order = request.GET.get('order', 'relevance')
+    context = dict(results=results, previous_page=previous_page, query=query,next_page=next_page, order=order)
+    return context
+
+
 @register.inclusion_tag(takes_context=True, filename='search/search_bar.html')
 def search_bar(context, tags=False, users=False):
     search_url = reverse('tags_list') if tags else reverse('community_list') if users else reverse('post_search')

biostar/forum/views.py

@@ -83,7 +83,7 @@ class CachedPaginator(Paginator):
     """
 
     # Time to live for the cache, in seconds
-    TTL = 3000
+    TTL = 300
 
     def __init__(self, cache_key='', ttl=None, *args, **kwargs):
         self.cache_key = cache_key
@@ -188,21 +188,23 @@ def get_posts(request, topic=""):
 def post_search(request):
     query = request.GET.get('query', '')
     length = len(query.replace(" ", ""))
+    page = int(request.GET.get('page', 1))
+    order = request.GET.get('order', 'relevance')
+
+    mapper = dict(relevance=None, date=['lastedit_date'])
+    sortedby = mapper.get(order)
 
     if length < settings.SEARCH_CHAR_MIN:
         messages.error(request, "Enter more characters before preforming search.")
         return redirect(reverse('post_list'))
 
-    results = search.perform_search(query=query)
-
-    total = len(results)
-    template_name = "search/search_results.html"
+    # Reverse sort when ordering by date.
+    revsort = order == 'date'
+    results, indexed = search.perform_search(query=query, page=page, reverse=revsort, sortedby=sortedby)
 
-    question_flag = Post.QUESTION
-    context = dict(results=results, query=query, total=total, template_name=template_name,
-                   question_flag=question_flag)
+    context = dict(results=results, query=query, indexed=indexed, order=order)
 
-    return render(request, template_name=template_name, context=context)
+    return render(request, "search/search_results.html", context=context)
 
 
 @check_params(allowed=ALLOWED_PARAMS)

themes/bioconductor/templates/forum_base.html

@@ -2,8 +2,6 @@
 {% load forum_tags %}
 {% load static %}
 
-{% block title %}Bioconductor Forum{% endblock %}
-
 {% block favicon %}
     <link rel="icon" href="{% static 'favicon.ico' %}" type="image/x-icon"/>
 {% endblock %}