Refactoring search to take advantage of ListableMixin fields.

docs/changelog.md

@@ -31,6 +31,9 @@
 
   * Text plugin now has a style setting
 
+  * Overhaul of search system, using a consistent approach to index and depict
+    rich/publishable/polymorphic models.
+
 ### Breaking changes:
 
   * `AbstractLayoutPage` now includes ListableMixin and HeroMixin. All models

docs/topics/search.md

@@ -0,0 +1,51 @@
+# Search in ICEkit
+
+ICEkit uses Haystack with an Elastic Search backend for its onsite search.
+
+It comes with a simple SearchIndex, `icekit.utils.search.AbstractLayoutIndex`
+which works well on content that:
+
+ * Implements the `ListableMixin` for getting list content
+ * Uses fluent contents (having a number of rich content placeholders)
+ * Is publishable (the index queryset defaults to `objects.published()`)
+ * Is polymorphic (the `django_ct` value always matches `get_model()`, rather
+   than varying by the object being indexed.
+
+ICEkit and GLAMkit's default content types use this index:
+
+ * Page (including LayoutPage, ArticleListingPage, etc.)
+ * Article
+ * Author
+
+And in optional libraries:
+ * icekit_events.EventBase
+ * icekit_press_releases.PressRelease
+ * glamkit_collections.WorkBase
+ * glamkit_collections.CreatorBase
+ * etc.
+
+`AbstractLayoutIndex` renders the `search/indexes/icekit/default.txt` template
+which indexes ListableMixin content, all Fluent placeholders, and some common
+content fields. You can render a different template in your search index by
+redeclaring the `title` field. HTML tags are stripped and HTML entities are
+converted to unicode.
+
+## Using `AbstractLayoutIndex`
+
+As minimal example, create `search_indexes.py` on a publishable, `ListableMixin`
+model:
+
+    from haystack import indexes
+    from icekit.utils.search import AbstractLayoutIndex
+    from . import models
+
+    class MyModelIndex(AbstractLayoutIndex, indexes.Indexable):
+        def get_model(self):
+            return models.MyModel
+
+Publish the content you want to be indexed, then run `manage.py update_index`.
+
+## Search Page
+
+The `icekit.page_types.search_page` page plugin implements a search page.
+To use it for your site, create a search page, and preview/publish it.

icekit/page_types/article/search_indexes.py

@@ -0,0 +1,7 @@
+from haystack import indexes
+from icekit.utils.search import AbstractLayoutIndex
+from . import models
+
+class ArticleIndex(AbstractLayoutIndex, indexes.Indexable):
+    def get_model(self):
+        return models.Article

icekit/page_types/author/search_indexes.py

@@ -1,33 +1,7 @@
-# from haystack import indexes
-#
-# from django.conf import settings
-# from django.utils import translation
-#
-# from . import models
-#
-#
-# class AuthorIndex(indexes.SearchIndex, indexes.Indexable):
-#     """
-#     Search index for `Author`.
-#     """
-#     text = indexes.CharField(document=True, use_template=True)
-#     name = indexes.CharField(model_attr='title', boost=2.0)
-#     url = indexes.CharField(model_attr='get_absolute_url')
-#     has_url = indexes.BooleanField(model_attr='get_absolute_url')
-#     # We add this for autocomplete.
-#     content_auto = indexes.EdgeNgramField(model_attr='title')
-#
-#     def index_queryset(self, using=None):
-#         """ Index only published authors """
-#         # TODO Hack to activate the site language if none is yet active, to
-#         # avoid complaints about null language_code when traversing the
-#         # `parent` relationship -- should probably do this elsewhere?
-#         if not translation.get_language():
-#             translation.activate(settings.LANGUAGE_CODE)
-#         return self.get_model().objects.published()
-#
-#     def get_model(self):
-#         """
-#         Get the model for the search index.
-#         """
-#         return models.Author
+from haystack import indexes
+from icekit.utils.search import AbstractLayoutIndex
+from . import models
+
+class AuthorIndex(AbstractLayoutIndex, indexes.Indexable):
+    def get_model(self):
+        return models.Author

icekit/page_types/search_page/abstract_models.py

@@ -1,15 +1,16 @@
 from fluent_pages.integration.fluent_contents import FluentContentsPage
+from icekit.mixins import ListableMixin
 
 from icekit.publishing.models import PublishableFluentContentsPage
 
 
-class AbstractUnpublishableSearchPage(FluentContentsPage):
+class AbstractUnpublishableSearchPage(FluentContentsPage, ListableMixin):
     class Meta:
         abstract = True
         verbose_name = 'Search page'
 
 
-class AbstractSearchPage(PublishableFluentContentsPage):
+class AbstractSearchPage(PublishableFluentContentsPage, ListableMixin):
     class Meta:
         abstract = True
         verbose_name = 'Search page'

icekit/page_types/search_page/migrations/0004_auto_20161122_2121.py

@@ -0,0 +1,24 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('search_page', '0003_auto_20160810_1856'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='searchpage',
+            name='boosted_search_terms',
+            field=models.TextField(blank=True, help_text='Words (space-separated) added here are boosted in relevance for search results increasing the chance of this appearing higher in the search results.'),
+        ),
+        migrations.AddField(
+            model_name='searchpage',
+            name='list_image',
+            field=models.ImageField(upload_to=b'icekit/listable/list_image/', blank=True, help_text=b"image to use in listings. Default image is used if this isn't given"),
+        ),
+    ]

icekit/search_indexes.py

@@ -13,5 +13,11 @@ def get_model(self):
         return Page
 
     def index_queryset(self, using=None):
+        """
+        Index current language translation of published objects.
+
+        TODO: Find a way to index all translations of the given model, not just
+        the current site language's translation.
+        """
         translation.activate(settings.LANGUAGE_CODE)
         return self.get_model().objects.filter(status=UrlNode.PUBLISHED).select_related()

icekit/templates/search/indexes/icekit/default.txt

@@ -5,6 +5,7 @@
     {% filter unescape %}
         {% block text %}
             {{ object.get_type }}
+            {{ object.slug }}
             {{ object.get_title }}
             {{ object.get_subtitle }}
             {{ object.get_oneliner }}
@@ -18,7 +19,9 @@
                 {% render_placeholder placeholder %}
             {% endfor %}
 
-            {{ object.boosted_search_terms }}{# may not be necessary if this is included/boosted in the query #}
+            {{ object.meta_keywords }}
+            {{ object.meta_description }}
+            {{ object.meta_title }}
         {% endblock %}
      {% endfilter %}
 {% endfilter %}

icekit/utils/search.py

@@ -31,25 +31,23 @@ class AbstractLayoutIndex(indexes.SearchIndex):
     # Meta
     get_absolute_url = indexes.CharField(model_attr='get_absolute_url')
     get_list_image_url = indexes.CharField()
-    modification_date = indexes.DateTimeField(model_attr='modification_date')
-    language_code = indexes.CharField(model_attr='language_code')
+    modification_date = indexes.DateTimeField()
+    language_code = indexes.CharField()
 
     # SEO Translations
-    meta_keywords = indexes.CharField(model_attr='meta_keywords')
-    meta_description = indexes.CharField(model_attr='meta_description')
-    meta_title = indexes.CharField(model_attr='meta_title')
+    meta_keywords = indexes.CharField()
+    meta_description = indexes.CharField()
+    meta_title = indexes.CharField()
 
     # We add this for autocomplete.
     content_auto = indexes.EdgeNgramField(model_attr='get_title')
 
     def index_queryset(self, using=None):
         """
-        Index current language translation of published objects.
+        Index published objects.
 
-        TODO: Find a way to index all translations of the given model, not just
-        the current site language's translation.
         """
-        return self.get_model().objects.published().language()
+        return self.get_model().objects.published().select_related()
 
     def full_prepare(self, obj):
         """
@@ -70,6 +68,21 @@ def prepare_get_list_image_url(self, obj):
                 pass
         return ""
 
+    def prepare_modification_date(self, obj):
+        return getattr(obj, "modification_date", None)
+
+    def prepare_language_code(self, obj):
+        return getattr(obj, "language_code", None)
+
+    def prepare_meta_keywords(self, obj):
+        return getattr(obj, "meta_keywords", None)
+
+    def prepare_meta_description(self, obj):
+        return getattr(obj, "meta_description", None)
+
+    def prepare_meta_title(self, obj):
+        return getattr(obj, "meta_title", None)
+
 
 class ICEkitSearchForm(ModelSearchForm):
     """ Custom search form to use the indexed fields defined above """
@@ -79,20 +92,13 @@ def get_searchqueryset(self, query):
         Add non-document fields to search query set so a) they are searched
         when querying, and b) any customisations like `boost` are applied.
         """
-        # TODO Find a way to detect all indexed fields across models and
-        # automatically add them to this filter, instead of requiring explicit
-        # naming of every indexed field.
+        # TODO Find a way to detect all (or boosted) indexed fields across
+        # models and automatically add them to this filter, instead of
+        # requiring explicit naming of every field to use in the query.
         return self.searchqueryset.filter(
             SQ(content=AutoQuery(query)) |  # Search `text` document
             SQ(title=AutoQuery(query)) |
-            SQ(slug=AutoQuery(query)) |
-            SQ(author=AutoQuery(query)) |
-
-            SQ(boosted_search_terms=AutoQuery(query)) |
-
-            SQ(meta_keywords=AutoQuery(query)) |
-            SQ(meta_description=AutoQuery(query)) |
-            SQ(meta_title=AutoQuery(query))
+            SQ(boosted_search_terms=AutoQuery(query))
         )
 
     # TODO This is mostly a copy/paste of `haystack.forms:SearchForm.search`