Use elasticsearch-py instead of pyelasticsearch.

elasticsearch-py is the official Python client for Elasticsearch.

docs/backend_support.rst

@@ -50,7 +50,7 @@ Elasticsearch
 * Stored (non-indexed) fields
 * Highlighting
 * Spatial search
-* Requires: pyelasticsearch 0.4+ & Elasticsearch 0.17.7+
+* Requires: elasticsearch-py 0.4.3+ & Elasticsearch 0.17.7+
 
 Whoosh
 ------

docs/installing_search_engines.rst

@@ -152,17 +152,16 @@ locally. Modifications should be done in a YAML file, the stock one being
       logs: /usr/local/var/log
       data: /usr/local/var/data
 
-You'll also need an Elasticsearch binding: pyelasticsearch_ (**NOT**
-``pyes``). Place ``pyelasticsearch`` somewhere on your ``PYTHONPATH``
-(usually ``python setup.py install`` or ``pip install pyelasticsearch``).
+You'll also need an Elasticsearch binding: elasticsearch-py_ (**NOT**
+``pyes``). Place ``elasticsearch`` somewhere on your ``PYTHONPATH``
+(usually ``python setup.py install`` or ``pip install elasticsearch``).
 
-.. _pyelasticsearch: http://pypi.python.org/pypi/pyelasticsearch/
+.. _elasticsearch-py: http://pypi.python.org/pypi/elasticsearch/
 
 .. note::
 
-    ``pyelasticsearch`` has its own dependencies that aren't covered by
-    Haystack. You'll also need ``requests`` & ``simplejson`` for speedier
-    JSON construction/parsing.
+    ``elasticsearch`` has its own dependencies that aren't covered by
+    Haystack. You'll also need ``urllib3``.
 
 
 Whoosh

docs/python3.rst

@@ -25,7 +25,7 @@ The following backends are fully supported under Python 3. However, you may
 need to update these dependencies if you have a pre-existing setup.
 
 * Solr (pysolr>=3.1.0)
-* Elasticsearch (pyelasticsearch>=0.5)
+* Elasticsearch
 
 
 Partially Supported Backends

haystack/backends/elasticsearch_backend.py

@@ -16,13 +16,10 @@
 from haystack.utils import log as logging
 
 try:
-    import requests
+    import elasticsearch
+    from elasticsearch.helpers import bulk_index
 except ImportError:
-    raise MissingDependency("The 'elasticsearch' backend requires the installation of 'requests'.")
-try:
-    import pyelasticsearch
-except ImportError:
-    raise MissingDependency("The 'elasticsearch' backend requires the installation of 'pyelasticsearch'. Please refer to the documentation.")
+    raise MissingDependency("The 'elasticsearch' backend requires the installation of 'elasticsearch'. Please refer to the documentation.")
 
 
 DATETIME_REGEX = re.compile(
@@ -100,7 +97,7 @@ def __init__(self, connection_alias, **connection_options):
         if not 'INDEX_NAME' in connection_options:
             raise ImproperlyConfigured("You must specify a 'INDEX_NAME' in your settings for connection '%s'." % connection_alias)
 
-        self.conn = pyelasticsearch.ElasticSearch(connection_options['URL'], timeout=self.timeout)
+        self.conn = elasticsearch.Elasticsearch(connection_options['URL'], timeout=self.timeout)
         self.index_name = connection_options['INDEX_NAME']
         self.log = logging.getLogger('haystack')
         self.setup_complete = False
@@ -114,7 +111,7 @@ def setup(self):
         # during the ``update`` & if it doesn't match, we'll put the new
         # mapping.
         try:
-            self.existing_mapping = self.conn.get_mapping(index=self.index_name)
+            self.existing_mapping = self.conn.indices.get_mapping(index=self.index_name)
         except Exception:
             if not self.silently_fail:
                 raise
@@ -134,8 +131,8 @@ def setup(self):
         if current_mapping != self.existing_mapping:
             try:
                 # Make sure the index is there first.
-                self.conn.create_index(self.index_name, self.DEFAULT_SETTINGS)
-                self.conn.put_mapping(self.index_name, 'modelresult', current_mapping)
+                self.conn.indices.create(self.index_name, self.DEFAULT_SETTINGS)
+                self.conn.indices.put_mapping(index=self.index_name, doc_type='modelresult', body=current_mapping)
                 self.existing_mapping = current_mapping
             except Exception:
                 if not self.silently_fail:
@@ -147,7 +144,7 @@ def update(self, index, iterable, commit=True):
         if not self.setup_complete:
             try:
                 self.setup()
-            except (requests.RequestException, pyelasticsearch.ElasticHttpError) as e:
+            except elasticsearch.TransportError as e:
                 if not self.silently_fail:
                     raise
 
@@ -164,9 +161,10 @@ def update(self, index, iterable, commit=True):
                 # Convert the data to make sure it's happy.
                 for key, value in prepped_data.items():
                     final_data[key] = self._from_python(value)
+                final_data['_id'] = final_data[ID]
 
                 prepped_docs.append(final_data)
-            except (requests.RequestException, pyelasticsearch.ElasticHttpError) as e:
+            except elasticsearch.TransportError as e:
                 if not self.silently_fail:
                     raise
 
@@ -180,36 +178,30 @@ def update(self, index, iterable, commit=True):
                     }
                 })
 
-        try:
-            self.conn.bulk_index(self.index_name, 'modelresult', prepped_docs, id_field=ID)
-        except ValueError:
-            # pyelasticsearch raise a ValueError exception
-            # when prepped_docs is empty
-            if not self.silently_fail:
-                raise
+        bulk_index(self.conn, prepped_docs, index=self.index_name, doc_type='modelresult')
 
         if commit:
-            self.conn.refresh(index=self.index_name)
+            self.conn.indices.refresh(index=self.index_name)
 
     def remove(self, obj_or_string, commit=True):
         doc_id = get_identifier(obj_or_string)
 
         if not self.setup_complete:
             try:
                 self.setup()
-            except (requests.RequestException, pyelasticsearch.ElasticHttpError) as e:
+            except elasticsearch.TransportError as e:
                 if not self.silently_fail:
                     raise
 
                 self.log.error("Failed to remove document '%s' from Elasticsearch: %s", doc_id, e)
                 return
 
         try:
-            self.conn.delete(self.index_name, 'modelresult', doc_id)
+            self.conn.delete(index=self.index_name, doc_type='modelresult', id=doc_id)
 
             if commit:
-                self.conn.refresh(index=self.index_name)
-        except (requests.RequestException, pyelasticsearch.ElasticHttpError) as e:
+                self.conn.indices.refresh(index=self.index_name)
+        except elasticsearch.TransportError as e:
             if not self.silently_fail:
                 raise
 
@@ -223,7 +215,7 @@ def clear(self, models=[], commit=True):
 
         try:
             if not models:
-                self.conn.delete_index(self.index_name)
+                self.conn.indices.delete(index=self.index_name)
                 self.setup_complete = False
                 self.existing_mapping = {}
             else:
@@ -235,8 +227,8 @@ def clear(self, models=[], commit=True):
                 # Delete by query in Elasticsearch asssumes you're dealing with
                 # a ``query`` root object. :/
                 query = {'query_string': {'query': " OR ".join(models_to_delete)}}
-                self.conn.delete_by_query(self.index_name, 'modelresult', query)
-        except (requests.RequestException, pyelasticsearch.ElasticHttpError) as e:
+                self.conn.delete_by_query(index=self.index_name, doc_type='modelresult', body=query)
+        except elasticsearch.TransportError as e:
             if not self.silently_fail:
                 raise
 
@@ -493,10 +485,10 @@ def search(self, query_string, **kwargs):
             search_kwargs['size'] = end_offset - start_offset
 
         try:
-            raw_results = self.conn.search(search_kwargs,
+            raw_results = self.conn.search(body=search_kwargs,
                                            index=self.index_name,
                                            doc_type='modelresult')
-        except (requests.RequestException, pyelasticsearch.ElasticHttpError) as e:
+        except elasticsearch.TransportError as e:
             if not self.silently_fail:
                 raise
 
@@ -533,8 +525,8 @@ def more_like_this(self, model_instance, additional_query_string=None,
         doc_id = get_identifier(model_instance)
 
         try:
-            raw_results = self.conn.more_like_this(self.index_name, 'modelresult', doc_id, [field_name], **params)
-        except (requests.RequestException, pyelasticsearch.ElasticHttpError) as e:
+            raw_results = self.conn.mlt(index=self.index_name, doc_type='modelresult', id=doc_id, mlt_fields=[field_name], **params)
+        except elasticsearch.TransportError as e:
             if not self.silently_fail:
                 raise
 

tests/core/tests/test_backends.py

@@ -27,9 +27,9 @@ def test_load_whoosh(self):
 
     def test_load_elasticsearch(self):
         try:
-            import pyelasticsearch
+            import elasticsearch
         except ImportError:
-            warnings.warn("Pyelasticsearch doesn't appear to be installed. Unable to test loading the ElasticSearch backend.")
+            warnings.warn("elasticsearch-py doesn't appear to be installed. Unable to test loading the ElasticSearch backend.")
             return
 
         backend = loading.load_backend('haystack.backends.elasticsearch_backend.ElasticsearchSearchEngine')

tests/elasticsearch_tests/tests/test_elasticsearch_backend.py

@@ -4,8 +4,7 @@
 import logging as std_logging
 import operator
 
-import pyelasticsearch
-import requests
+import elasticsearch
 from django.conf import settings
 from django.test import TestCase
 from django.utils import unittest
@@ -33,11 +32,11 @@
 
 def clear_elasticsearch_index():
     # Wipe it clean.
-    raw_es = pyelasticsearch.ElasticSearch(settings.HAYSTACK_CONNECTIONS['default']['URL'])
+    raw_es = elasticsearch.Elasticsearch(settings.HAYSTACK_CONNECTIONS['default']['URL'])
     try:
-        raw_es.delete_index(settings.HAYSTACK_CONNECTIONS['default']['INDEX_NAME'])
-        raw_es.refresh()
-    except (requests.RequestException, pyelasticsearch.ElasticHttpError):
+        raw_es.indices.delete(index=settings.HAYSTACK_CONNECTIONS['default']['INDEX_NAME'])
+        raw_es.indices.refresh()
+    except elasticsearch.TransportError:
         pass
 
 
@@ -204,7 +203,7 @@ def setUp(self):
         super(ElasticsearchSearchBackendTestCase, self).setUp()
 
         # Wipe it clean.
-        self.raw_es = pyelasticsearch.ElasticSearch(settings.HAYSTACK_CONNECTIONS['default']['URL'])
+        self.raw_es = elasticsearch.Elasticsearch(settings.HAYSTACK_CONNECTIONS['default']['URL'])
         clear_elasticsearch_index()
 
         # Stow.
@@ -235,8 +234,8 @@ def tearDown(self):
 
     def raw_search(self, query):
         try:
-            return self.raw_es.search('*:*', index=settings.HAYSTACK_CONNECTIONS['default']['INDEX_NAME'])
-        except (requests.RequestException, pyelasticsearch.ElasticHttpError):
+            return self.raw_es.search(q='*:*', index=settings.HAYSTACK_CONNECTIONS['default']['INDEX_NAME'])
+        except elasticsearch.TransportError:
             return {}
 
     def test_non_silent(self):
@@ -1226,7 +1225,7 @@ def setUp(self):
         super(ElasticsearchBoostBackendTestCase, self).setUp()
 
         # Wipe it clean.
-        self.raw_es = pyelasticsearch.ElasticSearch(settings.HAYSTACK_CONNECTIONS['default']['URL'])
+        self.raw_es = elasticsearch.Elasticsearch(settings.HAYSTACK_CONNECTIONS['default']['URL'])
         clear_elasticsearch_index()
 
         # Stow.
@@ -1258,7 +1257,7 @@ def tearDown(self):
         super(ElasticsearchBoostBackendTestCase, self).tearDown()
 
     def raw_search(self, query):
-        return self.raw_es.search('*:*', index=settings.HAYSTACK_CONNECTIONS['default']['INDEX_NAME'])
+        return self.raw_es.search(q='*:*', index=settings.HAYSTACK_CONNECTIONS['default']['INDEX_NAME'])
 
     def test_boost(self):
         self.sb.update(self.smmi, self.sample_objs)
@@ -1287,7 +1286,7 @@ def test__to_python(self):
 
 class RecreateIndexTestCase(TestCase):
     def setUp(self):
-        self.raw_es = pyelasticsearch.ElasticSearch(
+        self.raw_es = elasticsearch.Elasticsearch(
             settings.HAYSTACK_CONNECTIONS['default']['URL'])
 
     def test_recreate_index(self):
@@ -1297,14 +1296,14 @@ def test_recreate_index(self):
         sb.silently_fail = True
         sb.setup()
 
-        original_mapping = self.raw_es.get_mapping(sb.index_name)
+        original_mapping = self.raw_es.indices.get_mapping(index=sb.index_name)
 
         sb.clear()
         sb.setup()
 
         try:
-            updated_mapping = self.raw_es.get_mapping(sb.index_name)
-        except pyelasticsearch.ElasticHttpNotFoundError:
+            updated_mapping = self.raw_es.indices.get_mapping(sb.index_name)
+        except elasticsearch.NotFoundError:
             self.fail("There is no mapping after recreating the index")
         self.assertEqual(original_mapping, updated_mapping,
             "Mapping after recreating the index differs from the original one")

tests/requirements.txt

@@ -2,7 +2,7 @@ Django==1.4.2
 Whoosh==2.4.1
 httplib2==0.8
 mock==1.0.1
-pyelasticsearch==0.5
+elasticsearch==0.4.3
 pysolr==3.0.6
 pysqlite==2.6.3
 geopy==0.95.1

tox.ini

@@ -18,7 +18,7 @@ deps =
     pysolr==3.1.0
     poster
     whoosh==2.5.4
-    pyelasticsearch==0.6
+    elasticsearch==0.4.3
     httplib2==0.8
     python-dateutil
     geopy==0.95.1