diff --git a/docs/backend_support.rst b/docs/backend_support.rst index d62c088ac..07c747e5a 100644 --- a/docs/backend_support.rst +++ b/docs/backend_support.rst @@ -50,7 +50,7 @@ Elasticsearch * Stored (non-indexed) fields * Highlighting * Spatial search -* Requires: pyelasticsearch 0.4+ & Elasticsearch 0.17.7+ +* Requires: elasticsearch-py 0.4.3+ & Elasticsearch 0.17.7+ Whoosh ------ diff --git a/docs/installing_search_engines.rst b/docs/installing_search_engines.rst index 25fee9524..fdf375905 100644 --- a/docs/installing_search_engines.rst +++ b/docs/installing_search_engines.rst @@ -152,17 +152,16 @@ locally. Modifications should be done in a YAML file, the stock one being logs: /usr/local/var/log data: /usr/local/var/data -You'll also need an Elasticsearch binding: pyelasticsearch_ (**NOT** -``pyes``). Place ``pyelasticsearch`` somewhere on your ``PYTHONPATH`` -(usually ``python setup.py install`` or ``pip install pyelasticsearch``). +You'll also need an Elasticsearch binding: elasticsearch-py_ (**NOT** +``pyes``). Place ``elasticsearch`` somewhere on your ``PYTHONPATH`` +(usually ``python setup.py install`` or ``pip install elasticsearch``). -.. _pyelasticsearch: http://pypi.python.org/pypi/pyelasticsearch/ +.. _elasticsearch-py: http://pypi.python.org/pypi/elasticsearch/ .. note:: - ``pyelasticsearch`` has its own dependencies that aren't covered by - Haystack. You'll also need ``requests`` & ``simplejson`` for speedier - JSON construction/parsing. + ``elasticsearch`` has its own dependencies that aren't covered by + Haystack. You'll also need ``urllib3``. Whoosh diff --git a/docs/python3.rst b/docs/python3.rst index 23c288065..c4f364587 100644 --- a/docs/python3.rst +++ b/docs/python3.rst @@ -25,7 +25,7 @@ The following backends are fully supported under Python 3. However, you may need to update these dependencies if you have a pre-existing setup. * Solr (pysolr>=3.1.0) -* Elasticsearch (pyelasticsearch>=0.5) +* Elasticsearch Partially Supported Backends diff --git a/haystack/backends/elasticsearch_backend.py b/haystack/backends/elasticsearch_backend.py index 3d09a8c8d..9ed73fee2 100644 --- a/haystack/backends/elasticsearch_backend.py +++ b/haystack/backends/elasticsearch_backend.py @@ -16,13 +16,10 @@ from haystack.utils import log as logging try: - import requests + import elasticsearch + from elasticsearch.helpers import bulk_index except ImportError: - raise MissingDependency("The 'elasticsearch' backend requires the installation of 'requests'.") -try: - import pyelasticsearch -except ImportError: - raise MissingDependency("The 'elasticsearch' backend requires the installation of 'pyelasticsearch'. Please refer to the documentation.") + raise MissingDependency("The 'elasticsearch' backend requires the installation of 'elasticsearch'. Please refer to the documentation.") DATETIME_REGEX = re.compile( @@ -100,7 +97,7 @@ def __init__(self, connection_alias, **connection_options): if not 'INDEX_NAME' in connection_options: raise ImproperlyConfigured("You must specify a 'INDEX_NAME' in your settings for connection '%s'." % connection_alias) - self.conn = pyelasticsearch.ElasticSearch(connection_options['URL'], timeout=self.timeout) + self.conn = elasticsearch.Elasticsearch(connection_options['URL'], timeout=self.timeout) self.index_name = connection_options['INDEX_NAME'] self.log = logging.getLogger('haystack') self.setup_complete = False @@ -114,7 +111,7 @@ def setup(self): # during the ``update`` & if it doesn't match, we'll put the new # mapping. try: - self.existing_mapping = self.conn.get_mapping(index=self.index_name) + self.existing_mapping = self.conn.indices.get_mapping(index=self.index_name) except Exception: if not self.silently_fail: raise @@ -134,8 +131,8 @@ def setup(self): if current_mapping != self.existing_mapping: try: # Make sure the index is there first. - self.conn.create_index(self.index_name, self.DEFAULT_SETTINGS) - self.conn.put_mapping(self.index_name, 'modelresult', current_mapping) + self.conn.indices.create(self.index_name, self.DEFAULT_SETTINGS) + self.conn.indices.put_mapping(index=self.index_name, doc_type='modelresult', body=current_mapping) self.existing_mapping = current_mapping except Exception: if not self.silently_fail: @@ -147,7 +144,7 @@ def update(self, index, iterable, commit=True): if not self.setup_complete: try: self.setup() - except (requests.RequestException, pyelasticsearch.ElasticHttpError) as e: + except elasticsearch.TransportError as e: if not self.silently_fail: raise @@ -164,9 +161,10 @@ def update(self, index, iterable, commit=True): # Convert the data to make sure it's happy. for key, value in prepped_data.items(): final_data[key] = self._from_python(value) + final_data['_id'] = final_data[ID] prepped_docs.append(final_data) - except (requests.RequestException, pyelasticsearch.ElasticHttpError) as e: + except elasticsearch.TransportError as e: if not self.silently_fail: raise @@ -180,16 +178,10 @@ def update(self, index, iterable, commit=True): } }) - try: - self.conn.bulk_index(self.index_name, 'modelresult', prepped_docs, id_field=ID) - except ValueError: - # pyelasticsearch raise a ValueError exception - # when prepped_docs is empty - if not self.silently_fail: - raise + bulk_index(self.conn, prepped_docs, index=self.index_name, doc_type='modelresult') if commit: - self.conn.refresh(index=self.index_name) + self.conn.indices.refresh(index=self.index_name) def remove(self, obj_or_string, commit=True): doc_id = get_identifier(obj_or_string) @@ -197,7 +189,7 @@ def remove(self, obj_or_string, commit=True): if not self.setup_complete: try: self.setup() - except (requests.RequestException, pyelasticsearch.ElasticHttpError) as e: + except elasticsearch.TransportError as e: if not self.silently_fail: raise @@ -205,11 +197,11 @@ def remove(self, obj_or_string, commit=True): return try: - self.conn.delete(self.index_name, 'modelresult', doc_id) + self.conn.delete(index=self.index_name, doc_type='modelresult', id=doc_id) if commit: - self.conn.refresh(index=self.index_name) - except (requests.RequestException, pyelasticsearch.ElasticHttpError) as e: + self.conn.indices.refresh(index=self.index_name) + except elasticsearch.TransportError as e: if not self.silently_fail: raise @@ -223,7 +215,7 @@ def clear(self, models=[], commit=True): try: if not models: - self.conn.delete_index(self.index_name) + self.conn.indices.delete(index=self.index_name) self.setup_complete = False self.existing_mapping = {} else: @@ -235,8 +227,8 @@ def clear(self, models=[], commit=True): # Delete by query in Elasticsearch asssumes you're dealing with # a ``query`` root object. :/ query = {'query_string': {'query': " OR ".join(models_to_delete)}} - self.conn.delete_by_query(self.index_name, 'modelresult', query) - except (requests.RequestException, pyelasticsearch.ElasticHttpError) as e: + self.conn.delete_by_query(index=self.index_name, doc_type='modelresult', body=query) + except elasticsearch.TransportError as e: if not self.silently_fail: raise @@ -493,10 +485,10 @@ def search(self, query_string, **kwargs): search_kwargs['size'] = end_offset - start_offset try: - raw_results = self.conn.search(search_kwargs, + raw_results = self.conn.search(body=search_kwargs, index=self.index_name, doc_type='modelresult') - except (requests.RequestException, pyelasticsearch.ElasticHttpError) as e: + except elasticsearch.TransportError as e: if not self.silently_fail: raise @@ -533,8 +525,8 @@ def more_like_this(self, model_instance, additional_query_string=None, doc_id = get_identifier(model_instance) try: - raw_results = self.conn.more_like_this(self.index_name, 'modelresult', doc_id, [field_name], **params) - except (requests.RequestException, pyelasticsearch.ElasticHttpError) as e: + raw_results = self.conn.mlt(index=self.index_name, doc_type='modelresult', id=doc_id, mlt_fields=[field_name], **params) + except elasticsearch.TransportError as e: if not self.silently_fail: raise diff --git a/tests/core/tests/test_backends.py b/tests/core/tests/test_backends.py index 5923671ec..76fa7d4c4 100644 --- a/tests/core/tests/test_backends.py +++ b/tests/core/tests/test_backends.py @@ -27,9 +27,9 @@ def test_load_whoosh(self): def test_load_elasticsearch(self): try: - import pyelasticsearch + import elasticsearch except ImportError: - warnings.warn("Pyelasticsearch doesn't appear to be installed. Unable to test loading the ElasticSearch backend.") + warnings.warn("elasticsearch-py doesn't appear to be installed. Unable to test loading the ElasticSearch backend.") return backend = loading.load_backend('haystack.backends.elasticsearch_backend.ElasticsearchSearchEngine') diff --git a/tests/elasticsearch_tests/tests/test_elasticsearch_backend.py b/tests/elasticsearch_tests/tests/test_elasticsearch_backend.py index 058c31bc5..2c44f3c72 100644 --- a/tests/elasticsearch_tests/tests/test_elasticsearch_backend.py +++ b/tests/elasticsearch_tests/tests/test_elasticsearch_backend.py @@ -4,8 +4,7 @@ import logging as std_logging import operator -import pyelasticsearch -import requests +import elasticsearch from django.conf import settings from django.test import TestCase from django.utils import unittest @@ -33,11 +32,11 @@ def clear_elasticsearch_index(): # Wipe it clean. - raw_es = pyelasticsearch.ElasticSearch(settings.HAYSTACK_CONNECTIONS['default']['URL']) + raw_es = elasticsearch.Elasticsearch(settings.HAYSTACK_CONNECTIONS['default']['URL']) try: - raw_es.delete_index(settings.HAYSTACK_CONNECTIONS['default']['INDEX_NAME']) - raw_es.refresh() - except (requests.RequestException, pyelasticsearch.ElasticHttpError): + raw_es.indices.delete(index=settings.HAYSTACK_CONNECTIONS['default']['INDEX_NAME']) + raw_es.indices.refresh() + except elasticsearch.TransportError: pass @@ -204,7 +203,7 @@ def setUp(self): super(ElasticsearchSearchBackendTestCase, self).setUp() # Wipe it clean. - self.raw_es = pyelasticsearch.ElasticSearch(settings.HAYSTACK_CONNECTIONS['default']['URL']) + self.raw_es = elasticsearch.Elasticsearch(settings.HAYSTACK_CONNECTIONS['default']['URL']) clear_elasticsearch_index() # Stow. @@ -235,8 +234,8 @@ def tearDown(self): def raw_search(self, query): try: - return self.raw_es.search('*:*', index=settings.HAYSTACK_CONNECTIONS['default']['INDEX_NAME']) - except (requests.RequestException, pyelasticsearch.ElasticHttpError): + return self.raw_es.search(q='*:*', index=settings.HAYSTACK_CONNECTIONS['default']['INDEX_NAME']) + except elasticsearch.TransportError: return {} def test_non_silent(self): @@ -1226,7 +1225,7 @@ def setUp(self): super(ElasticsearchBoostBackendTestCase, self).setUp() # Wipe it clean. - self.raw_es = pyelasticsearch.ElasticSearch(settings.HAYSTACK_CONNECTIONS['default']['URL']) + self.raw_es = elasticsearch.Elasticsearch(settings.HAYSTACK_CONNECTIONS['default']['URL']) clear_elasticsearch_index() # Stow. @@ -1258,7 +1257,7 @@ def tearDown(self): super(ElasticsearchBoostBackendTestCase, self).tearDown() def raw_search(self, query): - return self.raw_es.search('*:*', index=settings.HAYSTACK_CONNECTIONS['default']['INDEX_NAME']) + return self.raw_es.search(q='*:*', index=settings.HAYSTACK_CONNECTIONS['default']['INDEX_NAME']) def test_boost(self): self.sb.update(self.smmi, self.sample_objs) @@ -1287,7 +1286,7 @@ def test__to_python(self): class RecreateIndexTestCase(TestCase): def setUp(self): - self.raw_es = pyelasticsearch.ElasticSearch( + self.raw_es = elasticsearch.Elasticsearch( settings.HAYSTACK_CONNECTIONS['default']['URL']) def test_recreate_index(self): @@ -1297,14 +1296,14 @@ def test_recreate_index(self): sb.silently_fail = True sb.setup() - original_mapping = self.raw_es.get_mapping(sb.index_name) + original_mapping = self.raw_es.indices.get_mapping(index=sb.index_name) sb.clear() sb.setup() try: - updated_mapping = self.raw_es.get_mapping(sb.index_name) - except pyelasticsearch.ElasticHttpNotFoundError: + updated_mapping = self.raw_es.indices.get_mapping(sb.index_name) + except elasticsearch.NotFoundError: self.fail("There is no mapping after recreating the index") self.assertEqual(original_mapping, updated_mapping, "Mapping after recreating the index differs from the original one") diff --git a/tests/requirements.txt b/tests/requirements.txt index 31bb6aeac..4b3bcef76 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -2,7 +2,7 @@ Django==1.4.2 Whoosh==2.4.1 httplib2==0.8 mock==1.0.1 -pyelasticsearch==0.5 +elasticsearch==0.4.3 pysolr==3.0.6 pysqlite==2.6.3 geopy==0.95.1 diff --git a/tox.ini b/tox.ini index dc22b5f44..4c16cd886 100644 --- a/tox.ini +++ b/tox.ini @@ -18,7 +18,7 @@ deps = pysolr==3.1.0 poster whoosh==2.5.4 - pyelasticsearch==0.6 + elasticsearch==0.4.3 httplib2==0.8 python-dateutil geopy==0.95.1