Merge pull request #673 from mitodl/feature/skm/669_elasticsearch_dsl

Adds elasticsearch-dsl and adds it alongside Haystack for now.

importer/api/__init__.py

@@ -25,8 +25,10 @@
     create_static_asset,
     get_video_sub,
     join_description_paths,
+    get_resources,
 )
 from learningresources.models import StaticAsset, course_asset_basepath
+from search.utils import index_resources
 
 log = logging.getLogger(__name__)
 
@@ -119,7 +121,6 @@ def import_course(bundle, repo_id, user_id, static_dir):
         learningresources.models.Course
     """
     src = bundle.course
-
     course = create_course(
         org=src.attrib["org"],
         repo_id=repo_id,
@@ -130,6 +131,10 @@ def import_course(bundle, repo_id, user_id, static_dir):
     import_static_assets(course, static_dir)
     import_children(course, src, None, '')
     populate_xanalytics_fields.delay(course.id)
+    # This triggers a bulk indexing of all LearningResource instances
+    # for the course at once.
+    index_resources(
+        get_resources(repo_id).filter(course__id=course.id))
     return course
 
 

learningresources/api.py

@@ -212,7 +212,7 @@ def get_resources(repo_id):
         list (list of learningresources.LearningResource): List of resources
     """
     return LearningResource.objects.select_related(
-        "learning_resource_type").filter(
+        "learning_resource_type", "course__repository").filter(
             course__repository__id=repo_id).order_by("title")
 
 

learningresources/tests/base.py

@@ -28,6 +28,7 @@
     update_description_path
 )
 from learningresources.models import Repository, StaticAsset
+from search.utils import clear_index, refresh_index
 
 log = logging.getLogger(__name__)
 # Using the md5 hasher speeds up tests.
@@ -86,6 +87,7 @@ def create_resource(self, **kwargs):
     def setUp(self):
         """set up"""
         super(LoreTestCase, self).setUp()
+        clear_index()
         self.user = User.objects.create_user(
             username=self.USERNAME, password=self.PASSWORD
         )
@@ -123,6 +125,7 @@ def setUp(self):
         self.client = Client()
 
         self.login(username=self.USERNAME)
+        refresh_index()
 
     def tearDown(self):
         """Clean up Elasticsearch and static assets between tests."""
@@ -131,6 +134,8 @@ def tearDown(self):
         for key, _ in haystack.connections.connections_info.items():
             haystack.connections.reload(key)
         call_command('clear_index', interactive=False, verbosity=0)
+        clear_index()
+        refresh_index()
 
     def _make_archive(self, path, make_zip=False, ext=None):
         """

requirements.txt

@@ -9,6 +9,7 @@ djangorestframework==3.2.2
 dj-database-url==0.3.0
 dj-static==0.0.6
 elasticsearch==1.6.0
+elasticsearch-dsl==0.0.8
 git+https://github.com/bpeschier/django-compressor-requirejs@889d5edc4f2acaa961c170084f1171c700649519#egg=githubdjango-compressor-requirejs-develop==0.4.1-odl
 psycopg2==2.6.1
 PyYAML==3.11

search/migrations/0001_initial.py

@@ -0,0 +1,30 @@
+# -*- coding: utf-8 -*-
+
+"""
+This migration should be able to be copy-pasted exactly for any
+future changes to the mapping, with the exception of the dependencies, which
+should be updated.
+"""
+
+from __future__ import unicode_literals
+
+from django.db import models, migrations
+
+from search.utils import create_mapping, index_resources
+
+# pylint: skip-file
+
+
+def create_learning_resource_mapping(apps, schema_editor):
+    create_mapping()
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('learningresources', '0016_revert_backfill'),
+        ('taxonomy', '0007_vocabulary_multi_terms'),
+    ]
+
+    operations = [
+        migrations.RunPython(create_learning_resource_mapping)
+    ]

search/signals.py

@@ -8,13 +8,11 @@
 
 import logging
 
-from django.db.models.signals import m2m_changed
+from django.db.models.signals import m2m_changed, post_save, post_delete
 from django.dispatch import receiver
 from haystack.signals import RealtimeSignalProcessor
 from statsd.defaults.django import statsd
 
-from search.search_indexes import LearningResourceIndex, get_vocabs
-
 log = logging.getLogger(__name__)
 
 
@@ -40,9 +38,38 @@ def handle_delete(self, sender, instance, **kwargs):
 @receiver(m2m_changed)
 def handle_m2m_save(sender, **kwargs):
     """Update index when taxonomies are updated."""
+    from search.search_indexes import LearningResourceIndex, get_vocabs
     instance = kwargs.pop("instance", None)
     if instance.__class__.__name__ != "LearningResource":
         return
     # Update cache for the LearningResource if it's already set.
     get_vocabs(instance.id)
     LearningResourceIndex().update_object(instance)
+    # Update Elasticsearch index:
+    from search.utils import index_resources
+    index_resources([instance])
+
+
+@statsd.timer('lore.elasticsearch.taxonomy_update')
+@receiver(post_save)
+def handle_resource_update(sender, **kwargs):
+    """Update index when a LearningResource is updated."""
+    if kwargs["created"]:
+        # Don't index upon create; update only.
+        return
+    instance = kwargs.pop("instance", None)
+    if instance.__class__.__name__ != "LearningResource":
+        return
+    from search.utils import index_resources
+    index_resources([instance])
+
+
+@statsd.timer('lore.elasticsearch.taxonomy_delete')
+@receiver(post_delete)
+def handle_resource_deletion(sender, **kwargs):
+    """Delete index when instance is deleted."""
+    instance = kwargs.pop("instance", None)
+    if instance.__class__.__name__ != "LearningResource":
+        return
+    from search.utils import delete_index
+    delete_index(instance)

search/tests/base_es.py

@@ -0,0 +1,54 @@
+# -*- coding: utf-8 -*-
+"""
+Tests for search engine indexing.
+
+Note that this file started out as a copy
+of search/tests/base.py, and was modified to use
+the Elasticsearch and Elasticsearch-DSL libraries instead of Haystack.
+"""
+from __future__ import unicode_literals
+
+import logging
+
+from learningresources.tests.base import LoreTestCase
+from search.utils import search_index
+from search.sorting import LoreSortingFields
+from taxonomy.models import Term, Vocabulary
+
+log = logging.getLogger(__name__)
+
+
+class SearchTestCase(LoreTestCase):
+    """Test Elasticsearch indexing."""
+
+    def setUp(self):
+        """Create a vocabulary for our tests."""
+        super(SearchTestCase, self).setUp()
+        self.vocabulary = Vocabulary.objects.create(
+            repository_id=self.repo.id, name="difficulty",
+            description="difficulty", required=False, vocabulary_type="f",
+            weight=1
+        )
+        self.terms = [
+            Term.objects.create(
+                vocabulary_id=self.vocabulary.id, label=label, weight=1
+            )
+            for label in ("easy", "medium", "very difficult", "ancòra", "very")
+        ]
+
+    def search(self, query, sorting=LoreSortingFields.DEFAULT_SORTING_FIELD):
+        """
+        Helper function to perform a search
+        """
+        return search_index(query, repo_slug=self.repo.slug, sort_by=sorting)
+
+    def count_results(self, query=None):
+        """Return count of matching indexed records."""
+        return self.search(query).count()
+
+    def count_faceted_results(self, vocab, term):
+        """Return count of matching indexed records by facet."""
+        return search_index(
+            repo_slug=self.repo.slug,
+            terms={vocab: term}
+        ).count()