Adds elasticsearch-dsl and adds it alongside Haystack for now.

Haystack will removed once all the features are added and tested. This requires
the addition of searching by facets, React.js updates, and updates to the
restful API.

importer/api/__init__.py

@@ -25,8 +25,10 @@
     create_static_asset,
     get_video_sub,
     join_description_paths,
+    get_resources,
 )
 from learningresources.models import StaticAsset, course_asset_basepath
+from search.utils import index_resources
 
 log = logging.getLogger(__name__)
 
@@ -119,7 +121,6 @@ def import_course(bundle, repo_id, user_id, static_dir):
         learningresources.models.Course
     """
     src = bundle.course
-
     course = create_course(
         org=src.attrib["org"],
         repo_id=repo_id,
@@ -130,6 +131,10 @@ def import_course(bundle, repo_id, user_id, static_dir):
     import_static_assets(course, static_dir)
     import_children(course, src, None, '')
     populate_xanalytics_fields.delay(course.id)
+    # This triggers a bulk indexing of all LearningResource instances
+    # for the course at once.
+    index_resources(
+        get_resources(repo_id).filter(course__id=course.id))
     return course
 
 

learningresources/api.py

@@ -212,7 +212,7 @@ def get_resources(repo_id):
         list (list of learningresources.LearningResource): List of resources
     """
     return LearningResource.objects.select_related(
-        "learning_resource_type").filter(
+        "learning_resource_type", "course__repository").filter(
             course__repository__id=repo_id).order_by("title")
 
 

learningresources/tests/base.py

@@ -28,6 +28,7 @@
     update_description_path
 )
 from learningresources.models import Repository, StaticAsset
+from search.utils import clear_index, refresh_index
 
 log = logging.getLogger(__name__)
 # Using the md5 hasher speeds up tests.
@@ -86,6 +87,7 @@ def create_resource(self, **kwargs):
     def setUp(self):
         """set up"""
         super(LoreTestCase, self).setUp()
+        clear_index()
         self.user = User.objects.create_user(
             username=self.USERNAME, password=self.PASSWORD
         )
@@ -123,6 +125,7 @@ def setUp(self):
         self.client = Client()
 
         self.login(username=self.USERNAME)
+        refresh_index()
 
     def tearDown(self):
         """Clean up Elasticsearch and static assets between tests."""
@@ -131,6 +134,8 @@ def tearDown(self):
         for key, _ in haystack.connections.connections_info.items():
             haystack.connections.reload(key)
         call_command('clear_index', interactive=False, verbosity=0)
+        clear_index()
+        refresh_index()
 
     def _make_archive(self, path, make_zip=False, ext=None):
         """

requirements.txt

@@ -9,6 +9,7 @@ djangorestframework==3.2.2
 dj-database-url==0.3.0
 dj-static==0.0.6
 elasticsearch==1.6.0
+elasticsearch-dsl==0.0.8
 git+https://github.com/bpeschier/django-compressor-requirejs@889d5edc4f2acaa961c170084f1171c700649519#egg=githubdjango-compressor-requirejs-develop==0.4.1-odl
 psycopg2==2.6.1
 PyYAML==3.11

search/migrations/0001_initial.py

@@ -0,0 +1,30 @@
+# -*- coding: utf-8 -*-
+
+"""
+This migration should be able to be copy-pasted exactly for any
+future changes to the mapping, with the exception of the dependencies, which
+should be updated.
+"""
+
+from __future__ import unicode_literals
+
+from django.db import models, migrations
+
+from search.utils import create_mapping, index_resources
+
+# pylint: skip-file
+
+
+def create_learning_resource_mapping(apps, schema_editor):
+    create_mapping()
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('learningresources', '0016_revert_backfill'),
+        ('taxonomy', '0007_vocabulary_multi_terms'),
+    ]
+
+    operations = [
+        migrations.RunPython(create_learning_resource_mapping)
+    ]

search/signals.py

@@ -8,13 +8,11 @@
 
 import logging
 
-from django.db.models.signals import m2m_changed
+from django.db.models.signals import m2m_changed, post_save, post_delete
 from django.dispatch import receiver
 from haystack.signals import RealtimeSignalProcessor
 from statsd.defaults.django import statsd
 
-from search.search_indexes import LearningResourceIndex, get_vocabs
-
 log = logging.getLogger(__name__)
 
 
@@ -40,9 +38,38 @@ def handle_delete(self, sender, instance, **kwargs):
 @receiver(m2m_changed)
 def handle_m2m_save(sender, **kwargs):
     """Update index when taxonomies are updated."""
+    from search.search_indexes import LearningResourceIndex, get_vocabs
     instance = kwargs.pop("instance", None)
     if instance.__class__.__name__ != "LearningResource":
         return
     # Update cache for the LearningResource if it's already set.
     get_vocabs(instance.id)
     LearningResourceIndex().update_object(instance)
+    # Update Elasticsearch index:
+    from search.utils import index_resources
+    index_resources([instance])
+
+
+@statsd.timer('lore.elasticsearch.taxonomy_update')
+@receiver(post_save)
+def handle_resource_update(sender, **kwargs):
+    """Update index when a LearningResource is updated."""
+    if kwargs["created"]:
+        # Don't index upon create; update only.
+        return
+    instance = kwargs.pop("instance", None)
+    if instance.__class__.__name__ != "LearningResource":
+        return
+    from search.utils import index_resources
+    index_resources([instance])
+
+
+@statsd.timer('lore.elasticsearch.taxonomy_delete')
+@receiver(post_delete)
+def handle_resource_deletion(sender, **kwargs):
+    """Delete index when instance is deleted."""
+    instance = kwargs.pop("instance", None)
+    if instance.__class__.__name__ != "LearningResource":
+        return
+    from search.utils import delete_index
+    delete_index(instance)

search/tests/base_es.py

@@ -0,0 +1,54 @@
+# -*- coding: utf-8 -*-
+"""
+Tests for search engine indexing.
+
+Note that this file started out as a copy
+of search/tests/base.py, and was modified to use
+the Elasticsearch and Elasticsearch-DSL libraries instead of Haystack.
+"""
+from __future__ import unicode_literals
+
+import logging
+
+from learningresources.tests.base import LoreTestCase
+from search.utils import search_index
+from search.sorting import LoreSortingFields
+from taxonomy.models import Term, Vocabulary
+
+log = logging.getLogger(__name__)
+
+
+class SearchTestCase(LoreTestCase):
+    """Test Elasticsearch indexing."""
+
+    def setUp(self):
+        """Create a vocabulary for our tests."""
+        super(SearchTestCase, self).setUp()
+        self.vocabulary = Vocabulary.objects.create(
+            repository_id=self.repo.id, name="difficulty",
+            description="difficulty", required=False, vocabulary_type="f",
+            weight=1
+        )
+        self.terms = [
+            Term.objects.create(
+                vocabulary_id=self.vocabulary.id, label=label, weight=1
+            )
+            for label in ("easy", "medium", "very difficult", "ancòra", "very")
+        ]
+
+    def search(self, query, sorting=LoreSortingFields.DEFAULT_SORTING_FIELD):
+        """
+        Helper function to perform a search
+        """
+        return search_index(query, repo_slug=self.repo.slug, sort_by=sorting)
+
+    def count_results(self, query=None):
+        """Return count of matching indexed records."""
+        return self.search(query).count()
+
+    def count_faceted_results(self, vocab, term):
+        """Return count of matching indexed records by facet."""
+        return search_index(
+            repo_slug=self.repo.slug,
+            terms={vocab: term}
+        ).count()