Permalink
Browse files

Updated elasticsearch backend to use a newer pyelasticsearch release …

…that features an improved API , connection pooling and better exception handling.
  • Loading branch information...
1 parent 3a31c5a commit 13e5a4926992581106ed14db1f8dd58a897c9a76 @jezdez jezdez committed Oct 6, 2012
View
@@ -50,7 +50,7 @@ Elasticsearch
* Stored (non-indexed) fields
* Highlighting
* Spatial search
-* Requires: pyelasticsearch 0.1+ & Elasticsearch 0.17.7+
+* Requires: pyelasticsearch 0.2+ & Elasticsearch 0.17.7+
Whoosh
------
@@ -153,10 +153,10 @@ locally. Modifications should be done in a YAML file, the stock one being
data: /usr/local/var/data
You'll also need an Elasticsearch binding: pyelasticsearch_ (**NOT**
-``pyes``). Place ``pyelasticsearch.py`` somewhere on your ``PYTHONPATH``
+``pyes``). Place ``pyelasticsearch`` somewhere on your ``PYTHONPATH``
(usually ``python setup.py install`` or ``pip install pyelasticsearch``).
-.. _pyelasticsearch: http://pypi.python.org/pypi/pyelasticsearch/0.1
+.. _pyelasticsearch: http://pypi.python.org/pypi/pyelasticsearch/
.. note::
@@ -61,26 +61,26 @@ class ElasticsearchSearchBackend(BaseSearchBackend):
"tokenizer": {
"haystack_ngram_tokenizer": {
"type": "nGram",
- "min_gram" : 3,
- "max_gram" : 15,
+ "min_gram": 3,
+ "max_gram": 15,
},
"haystack_edgengram_tokenizer": {
"type": "edgeNGram",
- "min_gram" : 2,
- "max_gram" : 15,
+ "min_gram": 2,
+ "max_gram": 15,
"side": "front"
}
},
- "filter" : {
- "haystack_ngram" : {
- "type" : "nGram",
- "min_gram" : 3,
- "max_gram" : 15
+ "filter": {
+ "haystack_ngram": {
+ "type": "nGram",
+ "min_gram": 3,
+ "max_gram": 15
},
- "haystack_edgengram" : {
- "type" : "edgeNGram",
- "min_gram" : 2,
- "max_gram" : 15
+ "haystack_edgengram": {
+ "type": "edgeNGram",
+ "min_gram": 2,
+ "max_gram": 15
}
}
}
@@ -110,8 +110,8 @@ def setup(self):
# during the ``update`` & if it doesn't match, we'll put the new
# mapping.
try:
- self.existing_mapping = self.conn.get_mapping(indexes=[self.index_name])
- except Exception, e:
+ self.existing_mapping = self.conn.get_mapping(index=self.index_name)
+ except Exception:
if not self.silently_fail:
raise
@@ -127,9 +127,9 @@ def setup(self):
try:
# Make sure the index is there first.
self.conn.create_index(self.index_name, self.DEFAULT_SETTINGS)
- self.conn.put_mapping('modelresult', current_mapping, indexes=[self.index_name])
+ self.conn.put_mapping('modelresult', current_mapping, index=self.index_name)
self.existing_mapping = current_mapping
- except Exception, e:
+ except Exception:
if not self.silently_fail:
raise
@@ -139,7 +139,7 @@ def update(self, index, iterable, commit=True):
if not self.setup_complete:
try:
self.setup()
- except pyelasticsearch.ElasticSearchError, e:
+ except (requests.RequestException, pyelasticsearch.ElasticHttpError), e:
if not self.silently_fail:
raise
@@ -158,7 +158,7 @@ def update(self, index, iterable, commit=True):
final_data[key] = self.conn.from_python(value)
prepped_docs.append(final_data)
- except (requests.RequestException, pyelasticsearch.ElasticSearchError), e:
+ except (requests.RequestException, pyelasticsearch.ElasticHttpError), e:
if not self.silently_fail:
raise
@@ -175,15 +175,15 @@ def update(self, index, iterable, commit=True):
self.conn.bulk_index(self.index_name, 'modelresult', prepped_docs, id_field=ID)
if commit:
- self.conn.refresh(indexes=[self.index_name])
+ self.conn.refresh(index=self.index_name)
def remove(self, obj_or_string, commit=True):
doc_id = get_identifier(obj_or_string)
if not self.setup_complete:
try:
self.setup()
- except pyelasticsearch.ElasticSearchError, e:
+ except (requests.RequestException, pyelasticsearch.ElasticHttpError), e:
if not self.silently_fail:
raise
@@ -194,8 +194,8 @@ def remove(self, obj_or_string, commit=True):
self.conn.delete(self.index_name, 'modelresult', doc_id)
if commit:
- self.conn.refresh(indexes=[self.index_name])
- except (requests.RequestException, pyelasticsearch.ElasticSearchError), e:
+ self.conn.refresh(index=self.index_name)
+ except (requests.RequestException, pyelasticsearch.ElasticHttpError), e:
if not self.silently_fail:
raise
@@ -218,11 +218,12 @@ def clear(self, models=[], commit=True):
# Delete by query in Elasticsearch asssumes you're dealing with
# a ``query`` root object. :/
- self.conn.delete_by_query(self.index_name, 'modelresult', {'query_string': {'query': " OR ".join(models_to_delete)}})
+ query = {'query_string': {'query': " OR ".join(models_to_delete)}}
+ self.conn.delete_by_query(self.index_name, 'modelresult', query)
if commit:
- self.conn.refresh(indexes=[self.index_name])
- except (requests.RequestException, pyelasticsearch.ElasticSearchError), e:
+ self.conn.refresh(index=self.index_name)
+ except (requests.RequestException, pyelasticsearch.ElasticHttpError), e:
if not self.silently_fail:
raise
@@ -474,25 +475,26 @@ def search(self, query_string, **kwargs):
self.setup()
search_kwargs = self.build_search_kwargs(query_string, **kwargs)
+ search_kwargs['from'] = kwargs.get('start_offset', 0)
- # Because Elasticsearch.
- query_params = {
- 'from': kwargs.get('start_offset', 0),
- }
-
- if kwargs.get('end_offset') is not None and kwargs.get('end_offset') > kwargs.get('start_offset', 0):
- query_params['size'] = kwargs.get('end_offset') - kwargs.get('start_offset', 0)
+ end_offset = kwargs.get('end_offset')
+ start_offset = kwargs.get('start_offset', 0)
+ if end_offset is not None and end_offset > start_offset:
+ search_kwargs['size'] = end_offset - start_offset
try:
- raw_results = self.conn.search(None, search_kwargs, indexes=[self.index_name], doc_types=['modelresult'], **query_params)
- except (requests.RequestException, pyelasticsearch.ElasticSearchError), e:
+ raw_results = self.conn.search(search_kwargs,
+ index=self.index_name,
+ doc_type='modelresult')
+ except (requests.RequestException, pyelasticsearch.ElasticHttpError), e:
if not self.silently_fail:
raise
self.log.error("Failed to query Elasticsearch using '%s': %s", query_string, e)
raw_results = {}
- return self._process_results(raw_results, highlight=kwargs.get('highlight'), result_class=kwargs.get('result_class', SearchResult))
+ return self._process_results(raw_results, highlight=kwargs.get('highlight'),
+ result_class=kwargs.get('result_class', SearchResult))
def more_like_this(self, model_instance, additional_query_string=None,
start_offset=0, end_offset=None, models=None,
@@ -521,8 +523,8 @@ def more_like_this(self, model_instance, additional_query_string=None,
doc_id = get_identifier(model_instance)
try:
- raw_results = self.conn.morelikethis(self.index_name, 'modelresult', doc_id, [field_name], **params)
- except (requests.RequestException, pyelasticsearch.ElasticSearchError), e:
+ raw_results = self.conn.more_like_this(self.index_name, 'modelresult', doc_id, [field_name], **params)
+ except (requests.RequestException, pyelasticsearch.ElasticHttpError), e:
if not self.silently_fail:
raise
@@ -834,7 +836,7 @@ def build_params(self, spelling_query=None, **kwargs):
search_kwargs['spelling_query'] = spelling_query
return search_kwargs
-
+
def run(self, spelling_query=None, **kwargs):
"""Builds and executes the query. Returns a list of search results."""
final_query = self.build_query()
@@ -4,9 +4,10 @@
import logging as std_logging
import pyelasticsearch
+import requests
from django.conf import settings
from django.test import TestCase
-from haystack import connections, connection_router, reset_search_queries
+from haystack import connections, reset_search_queries
from haystack import indexes
from haystack.inputs import AutoQuery
from haystack.models import SearchResult
@@ -35,7 +36,7 @@ def clear_elasticsearch_index():
try:
raw_es.delete_index(settings.HAYSTACK_CONNECTIONS['default']['INDEX_NAME'])
raw_es.refresh()
- except pyelasticsearch.ElasticSearchError:
+ except (requests.RequestException, pyelasticsearch.ElasticHttpError):
pass
@@ -195,8 +196,8 @@ def tearDown(self):
def raw_search(self, query):
try:
- return self.raw_es.search('*:*', indexes=[settings.HAYSTACK_CONNECTIONS['default']['INDEX_NAME']])
- except pyelasticsearch.ElasticSearchError:
+ return self.raw_es.search('*:*', index=settings.HAYSTACK_CONNECTIONS['default']['INDEX_NAME'])
+ except (requests.RequestException, pyelasticsearch.ElasticHttpError):
return {}
def test_non_silent(self):
@@ -231,7 +232,7 @@ def test_update(self):
# Check what Elasticsearch thinks is there.
self.assertEqual(self.raw_search('*:*')['hits']['total'], 3)
- self.assertEqual(sorted([res['_source'] for res in self.raw_search('*:*')['hits']['hits']], cmp=lambda x,y: cmp(x['id'], y['id'])), [
+ self.assertEqual(sorted([res['_source'] for res in self.raw_search('*:*')['hits']['hits']], cmp=lambda x, y: cmp(x['id'], y['id'])), [
{
'django_id': '1',
'django_ct': 'core.mockmodel',
@@ -1168,7 +1169,7 @@ def tearDown(self):
super(ElasticsearchBoostBackendTestCase, self).tearDown()
def raw_search(self, query):
- return self.raw_es.search('*:*', indexes=[settings.HAYSTACK_CONNECTIONS['default']['INDEX_NAME']])
+ return self.raw_es.search('*:*', index=settings.HAYSTACK_CONNECTIONS['default']['INDEX_NAME'])
def test_boost(self):
self.sb.update(self.smmi, self.sample_objs)

0 comments on commit 13e5a49

Please sign in to comment.