Merge pull request #3075 from robhudson/rm-elasticutils

Bug 1120526 - Remove all elasticutils

.gitmodules

@@ -76,9 +76,6 @@
 [submodule "vendor/src/elasticsearch-dsl-py"]
        path = vendor/src/elasticsearch-dsl-py
        url = git://github.com/elasticsearch/elasticsearch-dsl-py.git
-[submodule "vendor/src/elasticutils"]
-	path = vendor/src/elasticutils
-	url = git://github.com/mozilla/elasticutils.git
 [submodule "vendor/src/hash_ring"]
 	path = vendor/src/hash_ring
 	url = git://github.com/Doist/hash_ring.git

kuma/search/models.py

@@ -9,7 +9,6 @@
 from django.utils.functional import cached_property
 
 from elasticsearch.exceptions import NotFoundError, RequestError
-from elasticutils.contrib.django.tasks import index_objects
 
 from kuma.core.managers import PrefetchTaggableManager
 from kuma.core.urlresolvers import reverse
@@ -115,18 +114,18 @@ def record_outdated(self, instance):
                                                  content_object=instance)
 
     def promote(self):
-        rescheduled = []
+        from kuma.wiki.tasks import index_documents
+
+        # Index all outdated documents to this index.
+        outdated_ids = []
         for outdated_object in self.outdated_objects.all():
             instance = outdated_object.content_object
-            label = ('%s.%s.%s' %
-                     (outdated_object.content_type.natural_key() +
-                      (instance.id,)))  # gives us 'wiki.document.12345'
-            if label in rescheduled:
-                continue
-            mapping_type = instance.get_document_type()
-            index_objects.delay(mapping_type, [instance.id])
-            rescheduled.append(label)
+            outdated_ids.append(instance.id)
+        if outdated_ids:
+            index_documents.delay(outdated_ids, self.pk)
+        # Clear outdated.
         self.outdated_objects.all().delete()
+        # Promote this index.
         self.promoted = True
         self.save()
         # Allow only a single index to be promoted.

kuma/search/tasks.py

@@ -1,5 +1,4 @@
 import logging
-import warnings
 
 from django.conf import settings
 from django.core.mail import mail_admins
@@ -10,13 +9,6 @@
 log = logging.getLogger('kuma.search.tasks')
 
 
-# ignore a deprecation warning from elasticutils until the fix is released
-# refs https://github.com/mozilla/elasticutils/pull/160
-warnings.filterwarnings('ignore',
-                        category=DeprecationWarning,
-                        module='celery.decorators')
-
-
 @task
 def prepare_index(index_pk):
     """

kuma/search/tests/__init__.py

@@ -73,8 +73,8 @@ def tearDown(self):
         self.teardown_indexes()
         reset_url_prefixer()
 
-    def refresh(self, timesleep=0):
-        index = Index.objects.get_current().prefixed_name
+    def refresh(self, index=None, timesleep=0):
+        index = index or Index.objects.get_current().prefixed_name
         # Any time we're doing a refresh, we're making sure that the
         # index is ready to be queried.  Given that, it's almost
         # always the case that we want to run all the generated tasks,

kuma/search/tests/test_indexes.py

@@ -71,6 +71,9 @@ def test_outdated(self):
         # then create a successor and render a document against the old index
         successor_index = Index.objects.create(name='second')
         doc = Document.objects.get(pk=1)
+        doc.title = 'test outdated'
+        doc.slug = 'test-outdated'
+        doc.save()
         doc.render()
         eq_(successor_index.outdated_objects.count(), 1)
 
@@ -81,8 +84,14 @@ def test_outdated(self):
         eq_(S(index=successor_index.prefixed_name).count(), 7)
         eq_(S().query('match', title='lorem').execute()[0].slug, 'lorem-ipsum')
 
+        # Promotion reindexes outdated documents. Test that our change is
+        # reflected in the index.
         successor_index.promote()
+        self.refresh(index=successor_index.prefixed_name)
         eq_(successor_index.outdated_objects.count(), 0)
+        eq_(S(index=successor_index.prefixed_name)
+            .query('match', title='outdated').execute()[0].slug,
+            'test-outdated')
 
     def test_delete_index(self):
         # first create and populate the index

kuma/wiki/search.py

@@ -208,12 +208,13 @@ def get_index(cls):
 
     @classmethod
     def search(cls, **kwargs):
-        kwargs.update({
+        options = {
             'using': connections.get_connection(),
             'index': cls.get_index(),
             'doc_type': {cls._doc_type.name: cls.from_es},
-        })
-        sq = Search(**kwargs)
+        }
+        options.update(kwargs)
+        sq = Search(**options)
 
         # Add highlighting.
         sq = sq.highlight(*cls.excerpt_fields)
@@ -281,8 +282,8 @@ def reindex_all(cls, chunk_size=500, index=None, percent=100):
         """Rebuild ElasticSearch indexes.
 
         :arg chunk_size: how many documents to bulk index as a single chunk.
-        :arg index: the `Index` object to reindex into. Uses the current promoted
-            index if none provided.
+        :arg index: the `Index` object to reindex into. Uses the current
+            promoted index if none provided.
         :arg percent: 1 to 100--the percentage of the db to index.
 
         """

kuma/wiki/tasks.py

@@ -339,7 +339,9 @@ def index_documents(ids, index_pk, reraise=False):
             if reraise:
                 raise
 
-    cls.bulk_index(documents, id_field='id', es=es, index=index.prefixed_name)
+    if documents:
+        cls.bulk_index(documents, id_field='id', es=es,
+                       index=index.prefixed_name)
 
 
 @task

requirements/prod.txt

@@ -33,7 +33,6 @@ django-constance==0.6
 django-statici18n==0.4.5
 django-appconf==0.6
 urllib3==1.8
-elasticutils==0.9.1
 elasticsearch==0.4.5
 django-allauth==0.17.0
 requests-oauthlib==0.4.1

vendor/kuma.pth

@@ -69,7 +69,6 @@ src/pystatsd
 src/django-statsd
 src/elasticsearch-py
 src/elasticsearch-dsl-py
-src/elasticutils
 src/urllib3
 src/simplejson
 src/django-tidings