From 45cb407ba9e773efb812394aaac91740ca8d89fb Mon Sep 17 00:00:00 2001 From: joaojunior Date: Tue, 28 Jun 2016 21:01:35 +0000 Subject: [PATCH 01/20] Refactoring the code in pull request #1336 . This pull request is to permit use ElasticSearch 2.X --- haystack/backends/elasticsearch2_backend.py | 333 ++++ .../elasticsearch2_tests/__init__.py | 25 + .../elasticsearch2_tests/test_backend.py | 1498 +++++++++++++++++ .../elasticsearch2_tests/test_inputs.py | 85 + .../elasticsearch2_tests/test_query.py | 209 +++ test_haystack/elasticsearch_tests/__init__.py | 7 +- test_haystack/settings.py | 8 + 7 files changed, 2163 insertions(+), 2 deletions(-) create mode 100644 haystack/backends/elasticsearch2_backend.py create mode 100644 test_haystack/elasticsearch2_tests/__init__.py create mode 100644 test_haystack/elasticsearch2_tests/test_backend.py create mode 100644 test_haystack/elasticsearch2_tests/test_inputs.py create mode 100644 test_haystack/elasticsearch2_tests/test_query.py diff --git a/haystack/backends/elasticsearch2_backend.py b/haystack/backends/elasticsearch2_backend.py new file mode 100644 index 000000000..1e020ed81 --- /dev/null +++ b/haystack/backends/elasticsearch2_backend.py @@ -0,0 +1,333 @@ +# -*- coding: utf-8 -*- +from __future__ import absolute_import, division, print_function, unicode_literals + +import datetime + +from django.conf import settings + +from haystack.backends import BaseEngine +from haystack.backends.elasticsearch_backend import ElasticsearchSearchBackend, ElasticsearchSearchQuery +from haystack.constants import DJANGO_CT +from haystack.exceptions import MissingDependency +from haystack.utils import get_identifier, get_model_ct +from haystack.utils import log as logging + +try: + import elasticsearch + if not ((2, 0, 0) <= elasticsearch.__version__ < (3, 0, 0)): + raise ImportError + from elasticsearch.helpers import bulk, scan +except ImportError: + raise MissingDependency("The 'elasticsearch2' backend requires the \ + installation of 'elasticsearch>=2.0.0,<3.0.0'. \ + Please refer to the documentation.") + + +class Elasticsearch2SearchBackend(ElasticsearchSearchBackend): + def __init__(self, connection_alias, **connection_options): + super(Elasticsearch2SearchBackend, self).__init__(connection_alias, **connection_options) + self.content_field_name = None + + def clear(self, models=None, commit=True): + """ + Clears the backend of all documents/objects for a collection of models. + + :param models: List or tuple of models to clear. + :param commit: Not used. + """ + if models is not None: + assert isinstance(models, (list, tuple)) + + try: + if models is None: + self.conn.indices.delete(index=self.index_name, ignore=404) + self.setup_complete = False + self.existing_mapping = {} + self.content_field_name = None + else: + models_to_delete = [] + + for model in models: + models_to_delete.append("%s:%s" % (DJANGO_CT, get_model_ct(model))) + + # Delete using scroll API + query = {'query': {'query_string': {'query': " OR ".join(models_to_delete)}}} + generator = scan(self.conn, query=query, index=self.index_name, doc_type='modelresult') + actions = ({ + '_op_type': 'delete', + '_id': doc['_id'], + } for doc in generator) + bulk(self.conn, actions=actions, index=self.index_name, doc_type='modelresult') + self.conn.indices.refresh(index=self.index_name) + + except elasticsearch.TransportError as e: + if not self.silently_fail: + raise + + if models is not None: + self.log.error("Failed to clear Elasticsearch index of models '%s': %s", + ','.join(models_to_delete), e, exc_info=True) + else: + self.log.error("Failed to clear Elasticsearch index: %s", e, exc_info=True) + + def build_search_kwargs(self, query_string, sort_by=None, start_offset=0, end_offset=None, + fields='', highlight=False, facets=None, + date_facets=None, query_facets=None, + narrow_queries=None, spelling_query=None, + within=None, dwithin=None, distance_point=None, + models=None, limit_to_registered_models=None, + result_class=None): + kwargs = super(Elasticsearch2SearchBackend, self).build_search_kwargs(query_string, sort_by, + start_offset, end_offset, + fields, highlight, + spelling_query=spelling_query, + within=within, dwithin=dwithin, + distance_point=distance_point, + models=models, + limit_to_registered_models= + limit_to_registered_models, + result_class=result_class) + + filters = [] + if start_offset is not None: + kwargs['from'] = start_offset + + if end_offset is not None: + kwargs['size'] = end_offset - start_offset + + if narrow_queries is None: + narrow_queries = set() + + if facets is not None: + kwargs.setdefault('aggs', {}) + + for facet_fieldname, extra_options in facets.items(): + facet_options = { + 'meta': { + '_type': 'terms', + }, + 'terms': { + 'field': facet_fieldname, + } + } + if 'order' in extra_options: + facet_options['meta']['order'] = extra_options.pop('order') + # Special cases for options applied at the facet level (not the terms level). + if extra_options.pop('global_scope', False): + # Renamed "global_scope" since "global" is a python keyword. + facet_options['global'] = True + if 'facet_filter' in extra_options: + facet_options['facet_filter'] = extra_options.pop('facet_filter') + facet_options['terms'].update(extra_options) + kwargs['aggs'][facet_fieldname] = facet_options + + if date_facets is not None: + kwargs.setdefault('aggs', {}) + + for facet_fieldname, value in date_facets.items(): + # Need to detect on gap_by & only add amount if it's more than one. + interval = value.get('gap_by').lower() + + # Need to detect on amount (can't be applied on months or years). + if value.get('gap_amount', 1) != 1 and interval not in ('month', 'year'): + # Just the first character is valid for use. + interval = "%s%s" % (value['gap_amount'], interval[:1]) + + kwargs['aggs'][facet_fieldname] = { + 'meta': { + '_type': 'date_histogram', + }, + 'date_histogram': { + 'field': facet_fieldname, + 'interval': interval, + }, + 'aggs': { + facet_fieldname: { + 'date_range': { + 'field': facet_fieldname, + 'ranges': [ + { + 'from': self._from_python(value.get('start_date')), + 'to': self._from_python(value.get('end_date')), + } + ] + } + } + } + } + + if query_facets is not None: + kwargs.setdefault('aggs', {}) + + for facet_fieldname, value in query_facets: + kwargs['aggs'][facet_fieldname] = { + 'meta': { + '_type': 'query', + }, + 'filter': { + 'query_string': { + 'query': value, + } + }, + } + + for q in narrow_queries: + filters.append({ + 'query_string': { + 'query': q + } + }) + + # if we want to filter, change the query type to filteres + if filters: + kwargs["query"] = {"filtered": {"query": kwargs.pop("query")}} + filtered = kwargs["query"]["filtered"] + if 'filter' in filtered: + if "bool" in filtered["filter"].keys(): + another_filters = kwargs['query']['filtered']['filter']['bool']['must'] + else: + another_filters = [kwargs['query']['filtered']['filter']] + else: + another_filters = filters + + if len(another_filters) == 1: + kwargs['query']['filtered']["filter"] = another_filters[0] + else: + kwargs['query']['filtered']["filter"] = {"bool": {"must": another_filters}} + + return kwargs + + def more_like_this(self, model_instance, additional_query_string=None, + start_offset=0, end_offset=None, models=None, + limit_to_registered_models=None, result_class=None, **kwargs): + from haystack import connections + + if not self.setup_complete: + self.setup() + + # Deferred models will have a different class ("RealClass_Deferred_fieldname") + # which won't be in our registry: + model_klass = model_instance._meta.concrete_model + + index = connections[self.connection_alias].get_unified_index().get_index(model_klass) + field_name = index.get_content_field() + params = {} + + if start_offset is not None: + params['from_'] = start_offset + + if end_offset is not None: + params['size'] = end_offset - start_offset + + doc_id = get_identifier(model_instance) + + try: + # More like this Query + # https://www.elastic.co/guide/en/elasticsearch/reference/2.2/query-dsl-mlt-query.html + mlt_query = { + 'query': { + 'more_like_this': { + 'fields': [field_name], + 'like': [{ + "_id": doc_id + }] + } + } + } + + narrow_queries = [] + + if additional_query_string and additional_query_string != '*:*': + additional_filter = { + "query": { + "query_string": { + "query": additional_query_string + } + } + } + narrow_queries.append(additional_filter) + + if limit_to_registered_models is None: + limit_to_registered_models = getattr(settings, 'HAYSTACK_LIMIT_TO_REGISTERED_MODELS', True) + + if models and len(models): + model_choices = sorted(get_model_ct(model) for model in models) + elif limit_to_registered_models: + # Using narrow queries, limit the results to only models handled + # with the current routers. + model_choices = self.build_models_list() + else: + model_choices = [] + + if len(model_choices) > 0: + model_filter = {"terms": {DJANGO_CT: model_choices}} + narrow_queries.append(model_filter) + + if len(narrow_queries) > 0: + mlt_query = { + "query": { + "filtered": { + 'query': mlt_query['query'], + 'filter': { + 'bool': { + 'must': list(narrow_queries) + } + } + } + } + } + + raw_results = self.conn.search( + body=mlt_query, + index=self.index_name, + doc_type='modelresult', + _source=True, **params) + except elasticsearch.TransportError as e: + if not self.silently_fail: + raise + + self.log.error("Failed to fetch More Like This from Elasticsearch for document '%s': %s", + doc_id, e, exc_info=True) + raw_results = {} + + return self._process_results(raw_results, result_class=result_class) + + def _process_results(self, raw_results, highlight=False, + result_class=None, distance_point=None, + geo_sort=False): + results = super(Elasticsearch2SearchBackend, self)._process_results(raw_results, highlight, + result_class, distance_point, + geo_sort) + facets = {} + if 'aggregations' in raw_results: + facets = { + 'fields': {}, + 'dates': {}, + 'queries': {}, + } + + for facet_fieldname, facet_info in raw_results['aggregations'].items(): + facet_type = facet_info['meta']['_type'] + if facet_type == 'terms': + facets['fields'][facet_fieldname] = [(individual['key'], individual['doc_count']) for individual in facet_info['buckets']] + if 'order' in facet_info['meta']: + if facet_info['meta']['order'] == 'reverse_count': + srt = sorted(facets['fields'][facet_fieldname], key=lambda x: x[1]) + facets['fields'][facet_fieldname] = srt + elif facet_type == 'date_histogram': + # Elasticsearch provides UTC timestamps with an extra three + # decimals of precision, which datetime barfs on. + facets['dates'][facet_fieldname] = [(datetime.datetime.utcfromtimestamp(individual['key'] / 1000), individual['doc_count']) for individual in facet_info['buckets']] + elif facet_type == 'query': + facets['queries'][facet_fieldname] = facet_info['doc_count'] + results['facets'] = facets + return results + + +class Elasticsearch2SearchQuery(ElasticsearchSearchQuery): + pass + + +class Elasticsearch2SearchEngine(BaseEngine): + backend = Elasticsearch2SearchBackend + query = Elasticsearch2SearchQuery diff --git a/test_haystack/elasticsearch2_tests/__init__.py b/test_haystack/elasticsearch2_tests/__init__.py new file mode 100644 index 000000000..ba6384f46 --- /dev/null +++ b/test_haystack/elasticsearch2_tests/__init__.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +import warnings + +from django.conf import settings + +from ..utils import unittest + +warnings.simplefilter('ignore', Warning) + + +def setup(): + try: + import elasticsearch + if not ((2, 0, 0) <= elasticsearch.__version__ < (3, 0, 0)): + raise ImportError + from elasticsearch import Elasticsearch, exceptions + except ImportError: + raise unittest.SkipTest("'elasticsearch>=2.0.0,<3.0.0' not installed.") + + url = settings.HAYSTACK_CONNECTIONS['elasticsearch']['URL'] + es = Elasticsearch(url) + try: + es.info() + except exceptions.ConnectionError as e: + raise unittest.SkipTest("elasticsearch not running on %r" % url, e) diff --git a/test_haystack/elasticsearch2_tests/test_backend.py b/test_haystack/elasticsearch2_tests/test_backend.py new file mode 100644 index 000000000..14fd3b1aa --- /dev/null +++ b/test_haystack/elasticsearch2_tests/test_backend.py @@ -0,0 +1,1498 @@ +# -*- coding: utf-8 -*- +from __future__ import absolute_import, division, print_function, unicode_literals + +import datetime +import logging as std_logging +import operator +import unittest +from decimal import Decimal + +import elasticsearch +from django.apps import apps +from django.conf import settings +from django.test import TestCase +from django.test.utils import override_settings + +from haystack import connections, indexes, reset_search_queries +from haystack.exceptions import SkipDocument +from haystack.inputs import AutoQuery +from haystack.models import SearchResult +from haystack.query import RelatedSearchQuerySet, SearchQuerySet, SQ +from haystack.utils import log as logging +from haystack.utils.geo import Point +from haystack.utils.loading import UnifiedIndex +from ..core.models import AFourthMockModel, AnotherMockModel, ASixthMockModel, MockModel +from ..mocks import MockSearchResult + +test_pickling = True + +try: + import cPickle as pickle +except ImportError: + try: + import pickle + except ImportError: + test_pickling = False + + +def clear_elasticsearch_index(): + # Wipe it clean. + raw_es = elasticsearch.Elasticsearch(settings.HAYSTACK_CONNECTIONS['elasticsearch']['URL']) + try: + raw_es.indices.delete(index=settings.HAYSTACK_CONNECTIONS['elasticsearch']['INDEX_NAME']) + raw_es.indices.refresh() + except elasticsearch.TransportError: + pass + + # Since we've just completely deleted the index, we'll reset setup_complete so the next access will + # correctly define the mappings: + connections['elasticsearch'].get_backend().setup_complete = False + + +class Elasticsearch2MockSearchIndex(indexes.SearchIndex, indexes.Indexable): + text = indexes.CharField(document=True, use_template=True) + name = indexes.CharField(model_attr='author', faceted=True) + pub_date = indexes.DateTimeField(model_attr='pub_date') + + def get_model(self): + return MockModel + + +class Elasticsearch2MockSearchIndexWithSkipDocument(Elasticsearch2MockSearchIndex): + def prepare_text(self, obj): + if obj.author == 'daniel3': + raise SkipDocument + return u"Indexed!\n%s" % obj.id + + +class Elasticsearch2MockSpellingIndex(indexes.SearchIndex, indexes.Indexable): + text = indexes.CharField(document=True) + name = indexes.CharField(model_attr='author', faceted=True) + pub_date = indexes.DateTimeField(model_attr='pub_date') + + def get_model(self): + return MockModel + + def prepare_text(self, obj): + return obj.foo + + +class Elasticsearch2MaintainTypeMockSearchIndex(indexes.SearchIndex, indexes.Indexable): + text = indexes.CharField(document=True, use_template=True) + month = indexes.CharField(indexed=False) + pub_date = indexes.DateTimeField(model_attr='pub_date') + + def prepare_month(self, obj): + return "%02d" % obj.pub_date.month + + def get_model(self): + return MockModel + + +class Elasticsearch2MockModelSearchIndex(indexes.SearchIndex, indexes.Indexable): + text = indexes.CharField(model_attr='foo', document=True) + name = indexes.CharField(model_attr='author') + pub_date = indexes.DateTimeField(model_attr='pub_date') + + def get_model(self): + return MockModel + + +class Elasticsearch2AnotherMockModelSearchIndex(indexes.SearchIndex, indexes.Indexable): + text = indexes.CharField(document=True) + name = indexes.CharField(model_attr='author') + pub_date = indexes.DateTimeField(model_attr='pub_date') + + def get_model(self): + return AnotherMockModel + + def prepare_text(self, obj): + return u"You might be searching for the user %s" % obj.author + + +class Elasticsearch2BoostMockSearchIndex(indexes.SearchIndex, indexes.Indexable): + text = indexes.CharField( + document=True, use_template=True, + template_name='search/indexes/core/mockmodel_template.txt' + ) + author = indexes.CharField(model_attr='author', weight=2.0) + editor = indexes.CharField(model_attr='editor') + pub_date = indexes.DateTimeField(model_attr='pub_date') + + def get_model(self): + return AFourthMockModel + + def prepare(self, obj): + data = super(Elasticsearch2BoostMockSearchIndex, self).prepare(obj) + + if obj.pk == 4: + data['boost'] = 5.0 + + return data + + +class Elasticsearch2FacetingMockSearchIndex(indexes.SearchIndex, indexes.Indexable): + text = indexes.CharField(document=True) + author = indexes.CharField(model_attr='author', faceted=True) + editor = indexes.CharField(model_attr='editor', faceted=True) + pub_date = indexes.DateField(model_attr='pub_date', faceted=True) + facet_field = indexes.FacetCharField(model_attr='author') + + def prepare_text(self, obj): + return '%s %s' % (obj.author, obj.editor) + + def get_model(self): + return AFourthMockModel + + +class Elasticsearch2RoundTripSearchIndex(indexes.SearchIndex, indexes.Indexable): + text = indexes.CharField(document=True, default='') + name = indexes.CharField() + is_active = indexes.BooleanField() + post_count = indexes.IntegerField() + average_rating = indexes.FloatField() + price = indexes.DecimalField() + pub_date = indexes.DateField() + created = indexes.DateTimeField() + tags = indexes.MultiValueField() + sites = indexes.MultiValueField() + + def get_model(self): + return MockModel + + def prepare(self, obj): + prepped = super(Elasticsearch2RoundTripSearchIndex, self).prepare(obj) + prepped.update({ + 'text': 'This is some example text.', + 'name': 'Mister Pants', + 'is_active': True, + 'post_count': 25, + 'average_rating': 3.6, + 'price': Decimal('24.99'), + 'pub_date': datetime.date(2009, 11, 21), + 'created': datetime.datetime(2009, 11, 21, 21, 31, 00), + 'tags': ['staff', 'outdoor', 'activist', 'scientist'], + 'sites': [3, 5, 1], + }) + return prepped + + +class Elasticsearch2ComplexFacetsMockSearchIndex(indexes.SearchIndex, indexes.Indexable): + text = indexes.CharField(document=True, default='') + name = indexes.CharField(faceted=True) + is_active = indexes.BooleanField(faceted=True) + post_count = indexes.IntegerField() + post_count_i = indexes.FacetIntegerField(facet_for='post_count') + average_rating = indexes.FloatField(faceted=True) + pub_date = indexes.DateField(faceted=True) + created = indexes.DateTimeField(faceted=True) + sites = indexes.MultiValueField(faceted=True) + + def get_model(self): + return MockModel + + +class Elasticsearch2AutocompleteMockModelSearchIndex(indexes.SearchIndex, indexes.Indexable): + text = indexes.CharField(model_attr='foo', document=True) + name = indexes.CharField(model_attr='author') + pub_date = indexes.DateTimeField(model_attr='pub_date') + text_auto = indexes.EdgeNgramField(model_attr='foo') + name_auto = indexes.EdgeNgramField(model_attr='author') + + def get_model(self): + return MockModel + + +class Elasticsearch2SpatialSearchIndex(indexes.SearchIndex, indexes.Indexable): + text = indexes.CharField(model_attr='name', document=True) + location = indexes.LocationField() + + def prepare_location(self, obj): + return "%s,%s" % (obj.lat, obj.lon) + + def get_model(self): + return ASixthMockModel + + +class TestSettings(TestCase): + def test_kwargs_are_passed_on(self): + from haystack.backends.elasticsearch_backend import ElasticsearchSearchBackend + backend = ElasticsearchSearchBackend('alias', **{ + 'URL': settings.HAYSTACK_CONNECTIONS['elasticsearch']['URL'], + 'INDEX_NAME': 'testing', + 'KWARGS': {'max_retries': 42} + }) + + self.assertEqual(backend.conn.transport.max_retries, 42) + + +class Elasticsearch2SearchBackendTestCase(TestCase): + def setUp(self): + super(Elasticsearch2SearchBackendTestCase, self).setUp() + + # Wipe it clean. + self.raw_es = elasticsearch.Elasticsearch(settings.HAYSTACK_CONNECTIONS['elasticsearch']['URL']) + clear_elasticsearch_index() + + # Stow. + self.old_ui = connections['elasticsearch'].get_unified_index() + self.ui = UnifiedIndex() + self.smmi = Elasticsearch2MockSearchIndex() + self.smmidni = Elasticsearch2MockSearchIndexWithSkipDocument() + self.smtmmi = Elasticsearch2MaintainTypeMockSearchIndex() + self.ui.build(indexes=[self.smmi]) + connections['elasticsearch']._index = self.ui + self.sb = connections['elasticsearch'].get_backend() + + # Force the backend to rebuild the mapping each time. + self.sb.existing_mapping = {} + self.sb.setup() + + self.sample_objs = [] + + for i in range(1, 4): + mock = MockModel() + mock.id = i + mock.author = 'daniel%s' % i + mock.pub_date = datetime.date(2009, 2, 25) - datetime.timedelta(days=i) + self.sample_objs.append(mock) + + def tearDown(self): + connections['elasticsearch']._index = self.old_ui + super(Elasticsearch2SearchBackendTestCase, self).tearDown() + self.sb.silently_fail = True + + def raw_search(self, query): + try: + return self.raw_es.search(q='*:*', index=settings.HAYSTACK_CONNECTIONS['elasticsearch']['INDEX_NAME']) + except elasticsearch.TransportError: + return {} + + def test_non_silent(self): + bad_sb = connections['elasticsearch'].backend('bad', URL='http://omg.wtf.bbq:1000/', INDEX_NAME='whatver', + SILENTLY_FAIL=False, TIMEOUT=1) + + try: + bad_sb.update(self.smmi, self.sample_objs) + self.fail() + except: + pass + + try: + bad_sb.remove('core.mockmodel.1') + self.fail() + except: + pass + + try: + bad_sb.clear() + self.fail() + except: + pass + + try: + bad_sb.search('foo') + self.fail() + except: + pass + + def test_update_no_documents(self): + url = settings.HAYSTACK_CONNECTIONS['elasticsearch']['URL'] + index_name = settings.HAYSTACK_CONNECTIONS['elasticsearch']['INDEX_NAME'] + + sb = connections['elasticsearch'].backend('elasticsearch', URL=url, INDEX_NAME=index_name, SILENTLY_FAIL=True) + self.assertEqual(sb.update(self.smmi, []), None) + + sb = connections['elasticsearch'].backend('elasticsearch', URL=url, INDEX_NAME=index_name, + SILENTLY_FAIL=False) + try: + sb.update(self.smmi, []) + self.fail() + except: + pass + + def test_update(self): + self.sb.update(self.smmi, self.sample_objs) + + # Check what Elasticsearch thinks is there. + self.assertEqual(self.raw_search('*:*')['hits']['total'], 3) + self.assertEqual( + sorted([res['_source'] for res in self.raw_search('*:*')['hits']['hits']], key=lambda x: x['id']), [ + { + 'django_id': '1', + 'django_ct': 'core.mockmodel', + 'name': 'daniel1', + 'name_exact': 'daniel1', + 'text': 'Indexed!\n1', + 'pub_date': '2009-02-24T00:00:00', + 'id': 'core.mockmodel.1' + }, + { + 'django_id': '2', + 'django_ct': 'core.mockmodel', + 'name': 'daniel2', + 'name_exact': 'daniel2', + 'text': 'Indexed!\n2', + 'pub_date': '2009-02-23T00:00:00', + 'id': 'core.mockmodel.2' + }, + { + 'django_id': '3', + 'django_ct': 'core.mockmodel', + 'name': 'daniel3', + 'name_exact': 'daniel3', + 'text': 'Indexed!\n3', + 'pub_date': '2009-02-22T00:00:00', + 'id': 'core.mockmodel.3' + } + ]) + + def test_update_with_SkipDocument_raised(self): + self.sb.update(self.smmidni, self.sample_objs) + + # Check what Elasticsearch thinks is there. + res = self.raw_search('*:*')['hits'] + self.assertEqual(res['total'], 2) + self.assertListEqual( + sorted([x['_source']['id'] for x in res['hits']]), + ['core.mockmodel.1', 'core.mockmodel.2'] + ) + + def test_remove(self): + self.sb.update(self.smmi, self.sample_objs) + self.assertEqual(self.raw_search('*:*')['hits']['total'], 3) + + self.sb.remove(self.sample_objs[0]) + self.assertEqual(self.raw_search('*:*')['hits']['total'], 2) + self.assertEqual(sorted([res['_source'] for res in self.raw_search('*:*')['hits']['hits']], + key=operator.itemgetter('django_id')), [ + { + 'django_id': '2', + 'django_ct': 'core.mockmodel', + 'name': 'daniel2', + 'name_exact': 'daniel2', + 'text': 'Indexed!\n2', + 'pub_date': '2009-02-23T00:00:00', + 'id': 'core.mockmodel.2' + }, + { + 'django_id': '3', + 'django_ct': 'core.mockmodel', + 'name': 'daniel3', + 'name_exact': 'daniel3', + 'text': 'Indexed!\n3', + 'pub_date': '2009-02-22T00:00:00', + 'id': 'core.mockmodel.3' + } + ]) + + def test_remove_succeeds_on_404(self): + self.sb.silently_fail = False + self.sb.remove('core.mockmodel.421') + + def test_clear(self): + self.sb.update(self.smmi, self.sample_objs) + self.assertEqual(self.raw_search('*:*').get('hits', {}).get('total', 0), 3) + + self.sb.clear() + self.assertEqual(self.raw_search('*:*').get('hits', {}).get('total', 0), 0) + + self.sb.update(self.smmi, self.sample_objs) + self.assertEqual(self.raw_search('*:*').get('hits', {}).get('total', 0), 3) + + self.sb.clear([AnotherMockModel]) + self.assertEqual(self.raw_search('*:*').get('hits', {}).get('total', 0), 3) + + self.sb.clear([MockModel]) + self.assertEqual(self.raw_search('*:*').get('hits', {}).get('total', 0), 0) + + self.sb.update(self.smmi, self.sample_objs) + self.assertEqual(self.raw_search('*:*').get('hits', {}).get('total', 0), 3) + + self.sb.clear([AnotherMockModel, MockModel]) + self.assertEqual(self.raw_search('*:*').get('hits', {}).get('total', 0), 0) + + def test_search(self): + self.sb.update(self.smmi, self.sample_objs) + self.assertEqual(self.raw_search('*:*')['hits']['total'], 3) + + self.assertEqual(self.sb.search(''), {'hits': 0, 'results': []}) + self.assertEqual(self.sb.search('*:*')['hits'], 3) + self.assertEqual(set([result.pk for result in self.sb.search('*:*')['results']]), {u'2', u'1', u'3'}) + + self.assertEqual(self.sb.search('', highlight=True), {'hits': 0, 'results': []}) + self.assertEqual(self.sb.search('Index', highlight=True)['hits'], 3) + self.assertEqual( + sorted([result.highlighted[0] for result in self.sb.search('Index', highlight=True)['results']]), + [u'Indexed!\n1', u'Indexed!\n2', u'Indexed!\n3']) + + self.assertEqual(self.sb.search('Indx')['hits'], 0) + self.assertEqual(self.sb.search('indaxed')['spelling_suggestion'], 'indexed') + self.assertEqual(self.sb.search('arf', spelling_query='indexyd')['spelling_suggestion'], 'indexed') + + self.assertEqual(self.sb.search('', facets={'name': {}}), {'hits': 0, 'results': []}) + results = self.sb.search('Index', facets={'name': {}}) + self.assertEqual(results['hits'], 3) + self.assertSetEqual( + set(results['facets']['fields']['name']), + {('daniel3', 1), ('daniel2', 1), ('daniel1', 1)} + ) + + self.assertEqual(self.sb.search('', date_facets={ + 'pub_date': {'start_date': datetime.date(2008, 1, 1), 'end_date': datetime.date(2009, 4, 1), + 'gap_by': 'month', 'gap_amount': 1}}), {'hits': 0, 'results': []}) + results = self.sb.search('Index', date_facets={ + 'pub_date': {'start_date': datetime.date(2008, 1, 1), 'end_date': datetime.date(2009, 4, 1), + 'gap_by': 'month', 'gap_amount': 1}}) + self.assertEqual(results['hits'], 3) + self.assertEqual(results['facets']['dates']['pub_date'], [(datetime.datetime(2009, 2, 1, 0, 0), 3)]) + + self.assertEqual(self.sb.search('', query_facets=[('name', '[* TO e]')]), {'hits': 0, 'results': []}) + results = self.sb.search('Index', query_facets=[('name', '[* TO e]')]) + self.assertEqual(results['hits'], 3) + self.assertEqual(results['facets']['queries'], {u'name': 3}) + + self.assertEqual(self.sb.search('', narrow_queries={'name:daniel1'}), {'hits': 0, 'results': []}) + results = self.sb.search('Index', narrow_queries={'name:daniel1'}) + self.assertEqual(results['hits'], 1) + + # Ensure that swapping the ``result_class`` works. + self.assertTrue( + isinstance(self.sb.search(u'index', result_class=MockSearchResult)['results'][0], MockSearchResult)) + + # Check the use of ``limit_to_registered_models``. + self.assertEqual(self.sb.search('', limit_to_registered_models=False), {'hits': 0, 'results': []}) + self.assertEqual(self.sb.search('*:*', limit_to_registered_models=False)['hits'], 3) + self.assertEqual( + sorted([result.pk for result in self.sb.search('*:*', limit_to_registered_models=False)['results']]), + ['1', '2', '3']) + + # Stow. + old_limit_to_registered_models = getattr(settings, 'HAYSTACK_LIMIT_TO_REGISTERED_MODELS', True) + settings.HAYSTACK_LIMIT_TO_REGISTERED_MODELS = False + + self.assertEqual(self.sb.search(''), {'hits': 0, 'results': []}) + self.assertEqual(self.sb.search('*:*')['hits'], 3) + self.assertEqual(sorted([result.pk for result in self.sb.search('*:*')['results']]), ['1', '2', '3']) + + # Restore. + settings.HAYSTACK_LIMIT_TO_REGISTERED_MODELS = old_limit_to_registered_models + + def test_spatial_search_parameters(self): + p1 = Point(1.23, 4.56) + kwargs = self.sb.build_search_kwargs('*:*', distance_point={'field': 'location', 'point': p1}, + sort_by=(('distance', 'desc'),)) + + self.assertIn('sort', kwargs) + self.assertEqual(1, len(kwargs['sort'])) + geo_d = kwargs['sort'][0]['_geo_distance'] + + # ElasticSearch supports the GeoJSON-style lng, lat pairs so unlike Solr the values should be + # in the same order as we used to create the Point(): + # http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/query-dsl-geo-distance-filter.html#_lat_lon_as_array_4 + + self.assertDictEqual(geo_d, {'location': [1.23, 4.56], 'unit': 'km', 'order': 'desc'}) + + def test_more_like_this(self): + self.sb.update(self.smmi, self.sample_objs) + self.assertEqual(self.raw_search('*:*')['hits']['total'], 3) + + # A functional MLT example with enough data to work is below. Rely on + # this to ensure the API is correct enough. + self.assertEqual(self.sb.more_like_this(self.sample_objs[0])['hits'], 0) + self.assertEqual([result.pk for result in self.sb.more_like_this(self.sample_objs[0])['results']], []) + + def test_build_schema(self): + old_ui = connections['elasticsearch'].get_unified_index() + + (content_field_name, mapping) = self.sb.build_schema(old_ui.all_searchfields()) + self.assertEqual(content_field_name, 'text') + self.assertEqual(len(mapping), 4 + 2) # +2 management fields + self.assertEqual(mapping, { + 'django_id': {'index': 'not_analyzed', 'type': 'string', 'include_in_all': False}, + 'django_ct': {'index': 'not_analyzed', 'type': 'string', 'include_in_all': False}, + 'text': {'type': 'string', 'analyzer': 'snowball'}, + 'pub_date': {'type': 'date'}, + 'name': {'type': 'string', 'analyzer': 'snowball'}, + 'name_exact': {'index': 'not_analyzed', 'type': 'string'} + }) + + ui = UnifiedIndex() + ui.build(indexes=[Elasticsearch2ComplexFacetsMockSearchIndex()]) + (content_field_name, mapping) = self.sb.build_schema(ui.all_searchfields()) + self.assertEqual(content_field_name, 'text') + self.assertEqual(len(mapping), 15 + 2) # +2 management fields + self.assertEqual(mapping, { + 'django_id': {'index': 'not_analyzed', 'type': 'string', 'include_in_all': False}, + 'django_ct': {'index': 'not_analyzed', 'type': 'string', 'include_in_all': False}, + 'name': {'type': 'string', 'analyzer': 'snowball'}, + 'is_active_exact': {'type': 'boolean'}, + 'created': {'type': 'date'}, + 'post_count': {'type': 'long'}, + 'created_exact': {'type': 'date'}, + 'sites_exact': {'index': 'not_analyzed', 'type': 'string'}, + 'is_active': {'type': 'boolean'}, + 'sites': {'type': 'string', 'analyzer': 'snowball'}, + 'post_count_i': {'type': 'long'}, + 'average_rating': {'type': 'float'}, + 'text': {'type': 'string', 'analyzer': 'snowball'}, + 'pub_date_exact': {'type': 'date'}, + 'name_exact': {'index': 'not_analyzed', 'type': 'string'}, + 'pub_date': {'type': 'date'}, + 'average_rating_exact': {'type': 'float'} + }) + + def test_verify_type(self): + old_ui = connections['elasticsearch'].get_unified_index() + ui = UnifiedIndex() + smtmmi = Elasticsearch2MaintainTypeMockSearchIndex() + ui.build(indexes=[smtmmi]) + connections['elasticsearch']._index = ui + sb = connections['elasticsearch'].get_backend() + sb.update(smtmmi, self.sample_objs) + + self.assertEqual(sb.search('*:*')['hits'], 3) + self.assertEqual([result.month for result in sb.search('*:*')['results']], [u'02', u'02', u'02']) + connections['elasticsearch']._index = old_ui + + +class CaptureHandler(std_logging.Handler): + logs_seen = [] + + def emit(self, record): + CaptureHandler.logs_seen.append(record) + + +class FailedElasticsearch2SearchBackendTestCase(TestCase): + def setUp(self): + self.sample_objs = [] + + for i in range(1, 4): + mock = MockModel() + mock.id = i + mock.author = 'daniel%s' % i + mock.pub_date = datetime.date(2009, 2, 25) - datetime.timedelta(days=i) + self.sample_objs.append(mock) + + # Stow. + # Point the backend at a URL that doesn't exist so we can watch the + # sparks fly. + self.old_es_url = settings.HAYSTACK_CONNECTIONS['elasticsearch']['URL'] + settings.HAYSTACK_CONNECTIONS['elasticsearch']['URL'] = "%s/foo/" % self.old_es_url + self.cap = CaptureHandler() + logging.getLogger('haystack').addHandler(self.cap) + config = apps.get_app_config('haystack') + logging.getLogger('haystack').removeHandler(config.stream) + + # Setup the rest of the bits. + self.old_ui = connections['elasticsearch'].get_unified_index() + ui = UnifiedIndex() + self.smmi = Elasticsearch2MockSearchIndex() + ui.build(indexes=[self.smmi]) + connections['elasticsearch']._index = ui + self.sb = connections['elasticsearch'].get_backend() + + def tearDown(self): + # Restore. + settings.HAYSTACK_CONNECTIONS['elasticsearch']['URL'] = self.old_es_url + connections['elasticsearch']._index = self.old_ui + config = apps.get_app_config('haystack') + logging.getLogger('haystack').removeHandler(self.cap) + logging.getLogger('haystack').addHandler(config.stream) + + @unittest.expectedFailure + def test_all_cases(self): + # Prior to the addition of the try/except bits, these would all fail miserably. + self.assertEqual(len(CaptureHandler.logs_seen), 0) + + self.sb.update(self.smmi, self.sample_objs) + self.assertEqual(len(CaptureHandler.logs_seen), 1) + + self.sb.remove(self.sample_objs[0]) + self.assertEqual(len(CaptureHandler.logs_seen), 2) + + self.sb.search('search') + self.assertEqual(len(CaptureHandler.logs_seen), 3) + + self.sb.more_like_this(self.sample_objs[0]) + self.assertEqual(len(CaptureHandler.logs_seen), 4) + + self.sb.clear([MockModel]) + self.assertEqual(len(CaptureHandler.logs_seen), 5) + + self.sb.clear() + self.assertEqual(len(CaptureHandler.logs_seen), 6) + + +class LiveElasticsearch2SearchQueryTestCase(TestCase): + fixtures = ['base_data.json'] + + def setUp(self): + super(LiveElasticsearch2SearchQueryTestCase, self).setUp() + + # Wipe it clean. + clear_elasticsearch_index() + + # Stow. + self.old_ui = connections['elasticsearch'].get_unified_index() + self.ui = UnifiedIndex() + self.smmi = Elasticsearch2MockSearchIndex() + self.ui.build(indexes=[self.smmi]) + connections['elasticsearch']._index = self.ui + self.sb = connections['elasticsearch'].get_backend() + self.sq = connections['elasticsearch'].get_query() + + # Force indexing of the content. + self.smmi.update(using='elasticsearch') + + def tearDown(self): + connections['elasticsearch']._index = self.old_ui + super(LiveElasticsearch2SearchQueryTestCase, self).tearDown() + + def test_log_query(self): + reset_search_queries() + self.assertEqual(len(connections['elasticsearch'].queries), 0) + + with self.settings(DEBUG=False): + len(self.sq.get_results()) + self.assertEqual(len(connections['elasticsearch'].queries), 0) + + with self.settings(DEBUG=True): + # Redefine it to clear out the cached results. + self.sq = connections['elasticsearch'].query(using='elasticsearch') + self.sq.add_filter(SQ(name='bar')) + len(self.sq.get_results()) + self.assertEqual(len(connections['elasticsearch'].queries), 1) + self.assertEqual(connections['elasticsearch'].queries[0]['query_string'], + 'name:(bar)') + + # And again, for good measure. + self.sq = connections['elasticsearch'].query('elasticsearch') + self.sq.add_filter(SQ(name='bar')) + self.sq.add_filter(SQ(text='moof')) + len(self.sq.get_results()) + self.assertEqual(len(connections['elasticsearch'].queries), 2) + self.assertEqual(connections['elasticsearch'].queries[0]['query_string'], + 'name:(bar)') + self.assertEqual(connections['elasticsearch'].queries[1]['query_string'], + u'(name:(bar) AND text:(moof))') + + +lssqstc_all_loaded = None + + +@override_settings(DEBUG=True) +class LiveElasticsearch2SearchQuerySetTestCase(TestCase): + """Used to test actual implementation details of the SearchQuerySet.""" + fixtures = ['bulk_data.json'] + + def setUp(self): + super(LiveElasticsearch2SearchQuerySetTestCase, self).setUp() + + # Stow. + self.old_ui = connections['elasticsearch'].get_unified_index() + self.ui = UnifiedIndex() + self.smmi = Elasticsearch2MockSearchIndex() + self.ui.build(indexes=[self.smmi]) + connections['elasticsearch']._index = self.ui + + self.sqs = SearchQuerySet('elasticsearch') + self.rsqs = RelatedSearchQuerySet('elasticsearch') + + # Ugly but not constantly reindexing saves us almost 50% runtime. + global lssqstc_all_loaded + + if lssqstc_all_loaded is None: + lssqstc_all_loaded = True + + # Wipe it clean. + clear_elasticsearch_index() + + # Force indexing of the content. + self.smmi.update(using='elasticsearch') + + def tearDown(self): + # Restore. + connections['elasticsearch']._index = self.old_ui + super(LiveElasticsearch2SearchQuerySetTestCase, self).tearDown() + + def test_load_all(self): + sqs = self.sqs.order_by('pub_date').load_all() + self.assertTrue(isinstance(sqs, SearchQuerySet)) + self.assertTrue(len(sqs) > 0) + self.assertEqual(sqs[2].object.foo, + u'In addition, you may specify other fields to be populated along with the document. In this case, we also index the user who authored the document as well as the date the document was published. The variable you assign the SearchField to should directly map to the field your search backend is expecting. You instantiate most search fields with a parameter that points to the attribute of the object to populate that field with.') + + def test_iter(self): + reset_search_queries() + self.assertEqual(len(connections['elasticsearch'].queries), 0) + sqs = self.sqs.all() + results = sorted([int(result.pk) for result in sqs]) + self.assertEqual(results, list(range(1, 24))) + self.assertEqual(len(connections['elasticsearch'].queries), 3) + + def test_slice(self): + reset_search_queries() + self.assertEqual(len(connections['elasticsearch'].queries), 0) + results = self.sqs.all().order_by('pub_date') + self.assertEqual([int(result.pk) for result in results[1:11]], [3, 2, 4, 5, 6, 7, 8, 9, 10, 11]) + self.assertEqual(len(connections['elasticsearch'].queries), 1) + + reset_search_queries() + self.assertEqual(len(connections['elasticsearch'].queries), 0) + results = self.sqs.all().order_by('pub_date') + self.assertEqual(int(results[21].pk), 22) + self.assertEqual(len(connections['elasticsearch'].queries), 1) + + def test_values_slicing(self): + reset_search_queries() + self.assertEqual(len(connections['elasticsearch'].queries), 0) + + # TODO: this would be a good candidate for refactoring into a TestCase subclass shared across backends + + # The values will come back as strings because Hasytack doesn't assume PKs are integers. + # We'll prepare this set once since we're going to query the same results in multiple ways: + expected_pks = [str(i) for i in [3, 2, 4, 5, 6, 7, 8, 9, 10, 11]] + + results = self.sqs.all().order_by('pub_date').values('pk') + self.assertListEqual([i['pk'] for i in results[1:11]], expected_pks) + + results = self.sqs.all().order_by('pub_date').values_list('pk') + self.assertListEqual([i[0] for i in results[1:11]], expected_pks) + + results = self.sqs.all().order_by('pub_date').values_list('pk', flat=True) + self.assertListEqual(results[1:11], expected_pks) + + self.assertEqual(len(connections['elasticsearch'].queries), 3) + + def test_count(self): + reset_search_queries() + self.assertEqual(len(connections['elasticsearch'].queries), 0) + sqs = self.sqs.all() + self.assertEqual(sqs.count(), 23) + self.assertEqual(sqs.count(), 23) + self.assertEqual(len(sqs), 23) + self.assertEqual(sqs.count(), 23) + # Should only execute one query to count the length of the result set. + self.assertEqual(len(connections['elasticsearch'].queries), 1) + + def test_manual_iter(self): + results = self.sqs.all() + + reset_search_queries() + self.assertEqual(len(connections['elasticsearch'].queries), 0) + results = set([int(result.pk) for result in results._manual_iter()]) + self.assertEqual(results, + {2, 7, 12, 17, 1, 6, 11, 16, 23, 5, 10, 15, 22, 4, 9, 14, 19, 21, 3, 8, 13, 18, 20}) + self.assertEqual(len(connections['elasticsearch'].queries), 3) + + def test_fill_cache(self): + reset_search_queries() + self.assertEqual(len(connections['elasticsearch'].queries), 0) + results = self.sqs.all() + self.assertEqual(len(results._result_cache), 0) + self.assertEqual(len(connections['elasticsearch'].queries), 0) + results._fill_cache(0, 10) + self.assertEqual(len([result for result in results._result_cache if result is not None]), 10) + self.assertEqual(len(connections['elasticsearch'].queries), 1) + results._fill_cache(10, 20) + self.assertEqual(len([result for result in results._result_cache if result is not None]), 20) + self.assertEqual(len(connections['elasticsearch'].queries), 2) + + def test_cache_is_full(self): + reset_search_queries() + self.assertEqual(len(connections['elasticsearch'].queries), 0) + self.assertEqual(self.sqs._cache_is_full(), False) + results = self.sqs.all() + fire_the_iterator_and_fill_cache = [result for result in results] + self.assertEqual(results._cache_is_full(), True) + self.assertEqual(len(connections['elasticsearch'].queries), 3) + + def test___and__(self): + sqs1 = self.sqs.filter(content='foo') + sqs2 = self.sqs.filter(content='bar') + sqs = sqs1 & sqs2 + + self.assertTrue(isinstance(sqs, SearchQuerySet)) + self.assertEqual(len(sqs.query.query_filter), 2) + self.assertEqual(sqs.query.build_query(), u'((foo) AND (bar))') + + # Now for something more complex... + sqs3 = self.sqs.exclude(title='moof').filter(SQ(content='foo') | SQ(content='baz')) + sqs4 = self.sqs.filter(content='bar') + sqs = sqs3 & sqs4 + + self.assertTrue(isinstance(sqs, SearchQuerySet)) + self.assertEqual(len(sqs.query.query_filter), 3) + self.assertEqual(sqs.query.build_query(), u'(NOT (title:(moof)) AND ((foo) OR (baz)) AND (bar))') + + def test___or__(self): + sqs1 = self.sqs.filter(content='foo') + sqs2 = self.sqs.filter(content='bar') + sqs = sqs1 | sqs2 + + self.assertTrue(isinstance(sqs, SearchQuerySet)) + self.assertEqual(len(sqs.query.query_filter), 2) + self.assertEqual(sqs.query.build_query(), u'((foo) OR (bar))') + + # Now for something more complex... + sqs3 = self.sqs.exclude(title='moof').filter(SQ(content='foo') | SQ(content='baz')) + sqs4 = self.sqs.filter(content='bar').models(MockModel) + sqs = sqs3 | sqs4 + + self.assertTrue(isinstance(sqs, SearchQuerySet)) + self.assertEqual(len(sqs.query.query_filter), 2) + self.assertEqual(sqs.query.build_query(), u'((NOT (title:(moof)) AND ((foo) OR (baz))) OR (bar))') + + def test_auto_query(self): + # Ensure bits in exact matches get escaped properly as well. + # This will break horrifically if escaping isn't working. + sqs = self.sqs.auto_query('"pants:rule"') + self.assertTrue(isinstance(sqs, SearchQuerySet)) + self.assertEqual(repr(sqs.query.query_filter), '') + self.assertEqual(sqs.query.build_query(), u'("pants\\:rule")') + self.assertEqual(len(sqs), 0) + + # Regressions + + def test_regression_proper_start_offsets(self): + sqs = self.sqs.filter(text='index') + self.assertNotEqual(sqs.count(), 0) + + id_counts = {} + + for item in sqs: + if item.id in id_counts: + id_counts[item.id] += 1 + else: + id_counts[item.id] = 1 + + for key, value in id_counts.items(): + if value > 1: + self.fail("Result with id '%s' seen more than once in the results." % key) + + def test_regression_raw_search_breaks_slicing(self): + sqs = self.sqs.raw_search('text:index') + page_1 = [result.pk for result in sqs[0:10]] + page_2 = [result.pk for result in sqs[10:20]] + + for pk in page_2: + if pk in page_1: + self.fail("Result with id '%s' seen more than once in the results." % pk) + + # RelatedSearchQuerySet Tests + + def test_related_load_all(self): + sqs = self.rsqs.order_by('pub_date').load_all() + self.assertTrue(isinstance(sqs, SearchQuerySet)) + self.assertTrue(len(sqs) > 0) + self.assertEqual(sqs[2].object.foo, + u'In addition, you may specify other fields to be populated along with the document. In this case, we also index the user who authored the document as well as the date the document was published. The variable you assign the SearchField to should directly map to the field your search backend is expecting. You instantiate most search fields with a parameter that points to the attribute of the object to populate that field with.') + + def test_related_load_all_queryset(self): + sqs = self.rsqs.load_all().order_by('pub_date') + self.assertEqual(len(sqs._load_all_querysets), 0) + + sqs = sqs.load_all_queryset(MockModel, MockModel.objects.filter(id__gt=1)) + self.assertTrue(isinstance(sqs, SearchQuerySet)) + self.assertEqual(len(sqs._load_all_querysets), 1) + self.assertEqual(sorted([obj.object.id for obj in sqs]), list(range(2, 24))) + + sqs = sqs.load_all_queryset(MockModel, MockModel.objects.filter(id__gt=10)) + self.assertTrue(isinstance(sqs, SearchQuerySet)) + self.assertEqual(len(sqs._load_all_querysets), 1) + self.assertEqual(set([obj.object.id for obj in sqs]), {12, 17, 11, 16, 23, 15, 22, 14, 19, 21, 13, 18, 20}) + self.assertEqual(set([obj.object.id for obj in sqs[10:20]]), {21, 22, 23}) + + def test_related_iter(self): + reset_search_queries() + self.assertEqual(len(connections['elasticsearch'].queries), 0) + sqs = self.rsqs.all() + results = set([int(result.pk) for result in sqs]) + self.assertEqual(results, + {2, 7, 12, 17, 1, 6, 11, 16, 23, 5, 10, 15, 22, 4, 9, 14, 19, 21, 3, 8, 13, 18, 20}) + self.assertEqual(len(connections['elasticsearch'].queries), 3) + + def test_related_slice(self): + reset_search_queries() + self.assertEqual(len(connections['elasticsearch'].queries), 0) + results = self.rsqs.all().order_by('pub_date') + self.assertEqual([int(result.pk) for result in results[1:11]], [3, 2, 4, 5, 6, 7, 8, 9, 10, 11]) + self.assertEqual(len(connections['elasticsearch'].queries), 1) + + reset_search_queries() + self.assertEqual(len(connections['elasticsearch'].queries), 0) + results = self.rsqs.all().order_by('pub_date') + self.assertEqual(int(results[21].pk), 22) + self.assertEqual(len(connections['elasticsearch'].queries), 1) + + reset_search_queries() + self.assertEqual(len(connections['elasticsearch'].queries), 0) + results = self.rsqs.all().order_by('pub_date') + self.assertEqual(set([int(result.pk) for result in results[20:30]]), {21, 22, 23}) + self.assertEqual(len(connections['elasticsearch'].queries), 1) + + def test_related_manual_iter(self): + results = self.rsqs.all() + + reset_search_queries() + self.assertEqual(len(connections['elasticsearch'].queries), 0) + results = sorted([int(result.pk) for result in results._manual_iter()]) + self.assertEqual(results, list(range(1, 24))) + self.assertEqual(len(connections['elasticsearch'].queries), 3) + + def test_related_fill_cache(self): + reset_search_queries() + self.assertEqual(len(connections['elasticsearch'].queries), 0) + results = self.rsqs.all() + self.assertEqual(len(results._result_cache), 0) + self.assertEqual(len(connections['elasticsearch'].queries), 0) + results._fill_cache(0, 10) + self.assertEqual(len([result for result in results._result_cache if result is not None]), 10) + self.assertEqual(len(connections['elasticsearch'].queries), 1) + results._fill_cache(10, 20) + self.assertEqual(len([result for result in results._result_cache if result is not None]), 20) + self.assertEqual(len(connections['elasticsearch'].queries), 2) + + def test_related_cache_is_full(self): + reset_search_queries() + self.assertEqual(len(connections['elasticsearch'].queries), 0) + self.assertEqual(self.rsqs._cache_is_full(), False) + results = self.rsqs.all() + fire_the_iterator_and_fill_cache = [result for result in results] + self.assertEqual(results._cache_is_full(), True) + self.assertEqual(len(connections['elasticsearch'].queries), 3) + + def test_quotes_regression(self): + sqs = self.sqs.auto_query(u"44°48'40''N 20°28'32''E") + # Should not have empty terms. + self.assertEqual(sqs.query.build_query(), u"(44\xb048'40''N 20\xb028'32''E)") + # Should not cause Elasticsearch to 500. + self.assertEqual(sqs.count(), 0) + + sqs = self.sqs.auto_query('blazing') + self.assertEqual(sqs.query.build_query(), u'(blazing)') + self.assertEqual(sqs.count(), 0) + sqs = self.sqs.auto_query('blazing saddles') + self.assertEqual(sqs.query.build_query(), u'(blazing saddles)') + self.assertEqual(sqs.count(), 0) + sqs = self.sqs.auto_query('"blazing saddles') + self.assertEqual(sqs.query.build_query(), u'(\\"blazing saddles)') + self.assertEqual(sqs.count(), 0) + sqs = self.sqs.auto_query('"blazing saddles"') + self.assertEqual(sqs.query.build_query(), u'("blazing saddles")') + self.assertEqual(sqs.count(), 0) + sqs = self.sqs.auto_query('mel "blazing saddles"') + self.assertEqual(sqs.query.build_query(), u'(mel "blazing saddles")') + self.assertEqual(sqs.count(), 0) + sqs = self.sqs.auto_query('mel "blazing \'saddles"') + self.assertEqual(sqs.query.build_query(), u'(mel "blazing \'saddles")') + self.assertEqual(sqs.count(), 0) + sqs = self.sqs.auto_query('mel "blazing \'\'saddles"') + self.assertEqual(sqs.query.build_query(), u'(mel "blazing \'\'saddles")') + self.assertEqual(sqs.count(), 0) + sqs = self.sqs.auto_query('mel "blazing \'\'saddles"\'') + self.assertEqual(sqs.query.build_query(), u'(mel "blazing \'\'saddles" \')') + self.assertEqual(sqs.count(), 0) + sqs = self.sqs.auto_query('mel "blazing \'\'saddles"\'"') + self.assertEqual(sqs.query.build_query(), u'(mel "blazing \'\'saddles" \'\\")') + self.assertEqual(sqs.count(), 0) + sqs = self.sqs.auto_query('"blazing saddles" mel') + self.assertEqual(sqs.query.build_query(), u'("blazing saddles" mel)') + self.assertEqual(sqs.count(), 0) + sqs = self.sqs.auto_query('"blazing saddles" mel brooks') + self.assertEqual(sqs.query.build_query(), u'("blazing saddles" mel brooks)') + self.assertEqual(sqs.count(), 0) + sqs = self.sqs.auto_query('mel "blazing saddles" brooks') + self.assertEqual(sqs.query.build_query(), u'(mel "blazing saddles" brooks)') + self.assertEqual(sqs.count(), 0) + sqs = self.sqs.auto_query('mel "blazing saddles" "brooks') + self.assertEqual(sqs.query.build_query(), u'(mel "blazing saddles" \\"brooks)') + self.assertEqual(sqs.count(), 0) + + def test_query_generation(self): + sqs = self.sqs.filter(SQ(content=AutoQuery("hello world")) | SQ(title=AutoQuery("hello world"))) + self.assertEqual(sqs.query.build_query(), u"((hello world) OR title:(hello world))") + + def test_result_class(self): + # Assert that we're defaulting to ``SearchResult``. + sqs = self.sqs.all() + self.assertTrue(isinstance(sqs[0], SearchResult)) + + # Custom class. + sqs = self.sqs.result_class(MockSearchResult).all() + self.assertTrue(isinstance(sqs[0], MockSearchResult)) + + # Reset to default. + sqs = self.sqs.result_class(None).all() + self.assertTrue(isinstance(sqs[0], SearchResult)) + + +@override_settings(DEBUG=True) +class LiveElasticsearch2SpellingTestCase(TestCase): + """Used to test actual implementation details of the SearchQuerySet.""" + fixtures = ['bulk_data.json'] + + def setUp(self): + super(LiveElasticsearch2SpellingTestCase, self).setUp() + + # Stow. + self.old_ui = connections['elasticsearch'].get_unified_index() + self.ui = UnifiedIndex() + self.smmi = Elasticsearch2MockSpellingIndex() + self.ui.build(indexes=[self.smmi]) + connections['elasticsearch']._index = self.ui + + self.sqs = SearchQuerySet('elasticsearch') + + # Wipe it clean. + clear_elasticsearch_index() + + # Reboot the schema. + self.sb = connections['elasticsearch'].get_backend() + self.sb.setup() + + self.smmi.update(using='elasticsearch') + + def tearDown(self): + # Restore. + connections['elasticsearch']._index = self.old_ui + super(LiveElasticsearch2SpellingTestCase, self).tearDown() + + def test_spelling(self): + self.assertEqual(self.sqs.auto_query('structurd').spelling_suggestion(), 'structured') + self.assertEqual(self.sqs.spelling_suggestion('structurd'), 'structured') + self.assertEqual(self.sqs.auto_query('srchindex instanc').spelling_suggestion(), 'searchindex instance') + self.assertEqual(self.sqs.spelling_suggestion('srchindex instanc'), 'searchindex instance') + + +class LiveElasticsearch2MoreLikeThisTestCase(TestCase): + fixtures = ['bulk_data.json'] + + def setUp(self): + super(LiveElasticsearch2MoreLikeThisTestCase, self).setUp() + + # Wipe it clean. + clear_elasticsearch_index() + + self.old_ui = connections['elasticsearch'].get_unified_index() + self.ui = UnifiedIndex() + self.smmi = Elasticsearch2MockModelSearchIndex() + self.sammi = Elasticsearch2AnotherMockModelSearchIndex() + self.ui.build(indexes=[self.smmi, self.sammi]) + connections['elasticsearch']._index = self.ui + + self.sqs = SearchQuerySet('elasticsearch') + + self.smmi.update(using='elasticsearch') + self.sammi.update(using='elasticsearch') + + def tearDown(self): + # Restore. + connections['elasticsearch']._index = self.old_ui + super(LiveElasticsearch2MoreLikeThisTestCase, self).tearDown() + + def test_more_like_this(self): + mlt = self.sqs.more_like_this(MockModel.objects.get(pk=1)) + results = [result.pk for result in mlt] + self.assertEqual(mlt.count(), 11) + self.assertEqual(set(results), {u'10', u'5', u'2', u'21', u'4', u'6', u'23', u'9', u'14'}) + self.assertEqual(len(results), 10) + + alt_mlt = self.sqs.filter(name='daniel3').more_like_this(MockModel.objects.get(pk=2)) + results = [result.pk for result in alt_mlt] + self.assertEqual(alt_mlt.count(), 9) + self.assertEqual(set(results), {u'2', u'16', u'3', u'19', u'4', u'17', u'10', u'22', u'23'}) + self.assertEqual(len(results), 9) + + alt_mlt_with_models = self.sqs.models(MockModel).more_like_this(MockModel.objects.get(pk=1)) + results = [result.pk for result in alt_mlt_with_models] + self.assertEqual(alt_mlt_with_models.count(), 10) + self.assertEqual(set(results), {u'10', u'5', u'21', u'2', u'4', u'6', u'23', u'9', u'14', u'16'}) + self.assertEqual(len(results), 10) + + if hasattr(MockModel.objects, 'defer'): + # Make sure MLT works with deferred bits. + mi = MockModel.objects.defer('foo').get(pk=1) + self.assertEqual(mi._deferred, True) + deferred = self.sqs.models(MockModel).more_like_this(mi) + self.assertEqual(deferred.count(), 0) + self.assertEqual([result.pk for result in deferred], []) + self.assertEqual(len([result.pk for result in deferred]), 0) + + # Ensure that swapping the ``result_class`` works. + self.assertTrue( + isinstance(self.sqs.result_class(MockSearchResult).more_like_this(MockModel.objects.get(pk=1))[0], + MockSearchResult)) + + +class LiveElasticsearch2AutocompleteTestCase(TestCase): + fixtures = ['bulk_data.json'] + + def setUp(self): + super(LiveElasticsearch2AutocompleteTestCase, self).setUp() + + # Stow. + self.old_ui = connections['elasticsearch'].get_unified_index() + self.ui = UnifiedIndex() + self.smmi = Elasticsearch2AutocompleteMockModelSearchIndex() + self.ui.build(indexes=[self.smmi]) + connections['elasticsearch']._index = self.ui + + self.sqs = SearchQuerySet('elasticsearch') + + # Wipe it clean. + clear_elasticsearch_index() + + # Reboot the schema. + self.sb = connections['elasticsearch'].get_backend() + self.sb.setup() + + self.smmi.update(using='elasticsearch') + + def tearDown(self): + # Restore. + connections['elasticsearch']._index = self.old_ui + super(LiveElasticsearch2AutocompleteTestCase, self).tearDown() + + def test_build_schema(self): + self.sb = connections['elasticsearch'].get_backend() + content_name, mapping = self.sb.build_schema(self.ui.all_searchfields()) + self.assertEqual(mapping, { + 'django_id': {'index': 'not_analyzed', 'type': 'string', 'include_in_all': False}, + 'django_ct': {'index': 'not_analyzed', 'type': 'string', 'include_in_all': False}, + 'name_auto': { + 'type': 'string', + 'analyzer': 'edgengram_analyzer', + }, + 'text': { + 'type': 'string', + 'analyzer': 'snowball', + }, + 'pub_date': { + 'type': 'date' + }, + 'name': { + 'type': 'string', + 'analyzer': 'snowball', + }, + 'text_auto': { + 'type': 'string', + 'analyzer': 'edgengram_analyzer', + } + }) + + def test_autocomplete(self): + autocomplete = self.sqs.autocomplete(text_auto='mod') + self.assertEqual(autocomplete.count(), 16) + self.assertEqual(set([result.pk for result in autocomplete]), + {'1', '12', '6', '14', '7', '4', '23', '17', '13', '18', '20', '22', '19', '15', '10', '2'}) + self.assertTrue('mod' in autocomplete[0].text.lower()) + self.assertTrue('mod' in autocomplete[1].text.lower()) + self.assertTrue('mod' in autocomplete[2].text.lower()) + self.assertTrue('mod' in autocomplete[3].text.lower()) + self.assertTrue('mod' in autocomplete[4].text.lower()) + self.assertEqual(len([result.pk for result in autocomplete]), 16) + + # Test multiple words. + autocomplete_2 = self.sqs.autocomplete(text_auto='your mod') + self.assertEqual(autocomplete_2.count(), 13) + self.assertEqual(set([result.pk for result in autocomplete_2]), + {'1', '6', '2', '14', '12', '13', '10', '19', '4', '20', '23', '22', '15'}) + map_results = {result.pk: result for result in autocomplete_2} + self.assertTrue('your' in map_results['1'].text.lower()) + self.assertTrue('mod' in map_results['1'].text.lower()) + self.assertTrue('your' in map_results['6'].text.lower()) + self.assertTrue('mod' in map_results['6'].text.lower()) + self.assertTrue('your' in map_results['2'].text.lower()) + self.assertEqual(len([result.pk for result in autocomplete_2]), 13) + + # Test multiple fields. + autocomplete_3 = self.sqs.autocomplete(text_auto='Django', name_auto='dan') + self.assertEqual(autocomplete_3.count(), 4) + self.assertEqual(set([result.pk for result in autocomplete_3]), {'12', '1', '22', '14'}) + self.assertEqual(len([result.pk for result in autocomplete_3]), 4) + + # Test numbers in phrases + autocomplete_4 = self.sqs.autocomplete(text_auto='Jen 867') + self.assertEqual(autocomplete_4.count(), 1) + self.assertEqual(set([result.pk for result in autocomplete_4]), {'20'}) + + # Test numbers alone + autocomplete_4 = self.sqs.autocomplete(text_auto='867') + self.assertEqual(autocomplete_4.count(), 1) + self.assertEqual(set([result.pk for result in autocomplete_4]), {'20'}) + + +class LiveElasticsearch2RoundTripTestCase(TestCase): + def setUp(self): + super(LiveElasticsearch2RoundTripTestCase, self).setUp() + + # Wipe it clean. + clear_elasticsearch_index() + + # Stow. + self.old_ui = connections['elasticsearch'].get_unified_index() + self.ui = UnifiedIndex() + self.srtsi = Elasticsearch2RoundTripSearchIndex() + self.ui.build(indexes=[self.srtsi]) + connections['elasticsearch']._index = self.ui + self.sb = connections['elasticsearch'].get_backend() + + self.sqs = SearchQuerySet('elasticsearch') + + # Fake indexing. + mock = MockModel() + mock.id = 1 + self.sb.update(self.srtsi, [mock]) + + def tearDown(self): + # Restore. + connections['elasticsearch']._index = self.old_ui + super(LiveElasticsearch2RoundTripTestCase, self).tearDown() + + def test_round_trip(self): + results = self.sqs.filter(id='core.mockmodel.1') + + # Sanity check. + self.assertEqual(results.count(), 1) + + # Check the individual fields. + result = results[0] + self.assertEqual(result.id, 'core.mockmodel.1') + self.assertEqual(result.text, 'This is some example text.') + self.assertEqual(result.name, 'Mister Pants') + self.assertEqual(result.is_active, True) + self.assertEqual(result.post_count, 25) + self.assertEqual(result.average_rating, 3.6) + self.assertEqual(result.price, u'24.99') + self.assertEqual(result.pub_date, datetime.date(2009, 11, 21)) + self.assertEqual(result.created, datetime.datetime(2009, 11, 21, 21, 31, 00)) + self.assertEqual(result.tags, ['staff', 'outdoor', 'activist', 'scientist']) + self.assertEqual(result.sites, [3, 5, 1]) + + +@unittest.skipUnless(test_pickling, 'Skipping pickling tests') +class LiveElasticsearch2PickleTestCase(TestCase): + fixtures = ['bulk_data.json'] + + def setUp(self): + super(LiveElasticsearch2PickleTestCase, self).setUp() + + # Wipe it clean. + clear_elasticsearch_index() + + # Stow. + self.old_ui = connections['elasticsearch'].get_unified_index() + self.ui = UnifiedIndex() + self.smmi = Elasticsearch2MockModelSearchIndex() + self.sammi = Elasticsearch2AnotherMockModelSearchIndex() + self.ui.build(indexes=[self.smmi, self.sammi]) + connections['elasticsearch']._index = self.ui + + self.sqs = SearchQuerySet('elasticsearch') + + self.smmi.update(using='elasticsearch') + self.sammi.update(using='elasticsearch') + + def tearDown(self): + # Restore. + connections['elasticsearch']._index = self.old_ui + super(LiveElasticsearch2PickleTestCase, self).tearDown() + + def test_pickling(self): + results = self.sqs.all() + + for res in results: + # Make sure the cache is full. + pass + + in_a_pickle = pickle.dumps(results) + like_a_cuke = pickle.loads(in_a_pickle) + self.assertEqual(len(like_a_cuke), len(results)) + self.assertEqual(like_a_cuke[0].id, results[0].id) + + +class Elasticsearch2BoostBackendTestCase(TestCase): + def setUp(self): + super(Elasticsearch2BoostBackendTestCase, self).setUp() + + # Wipe it clean. + self.raw_es = elasticsearch.Elasticsearch(settings.HAYSTACK_CONNECTIONS['elasticsearch']['URL']) + clear_elasticsearch_index() + + # Stow. + self.old_ui = connections['elasticsearch'].get_unified_index() + self.ui = UnifiedIndex() + self.smmi = Elasticsearch2BoostMockSearchIndex() + self.ui.build(indexes=[self.smmi]) + connections['elasticsearch']._index = self.ui + self.sb = connections['elasticsearch'].get_backend() + + self.sample_objs = [] + + for i in range(1, 5): + mock = AFourthMockModel() + mock.id = i + + if i % 2: + mock.author = 'daniel' + mock.editor = 'david' + else: + mock.author = 'david' + mock.editor = 'daniel' + + mock.pub_date = datetime.date(2009, 2, 25) - datetime.timedelta(days=i) + self.sample_objs.append(mock) + + def tearDown(self): + connections['elasticsearch']._index = self.old_ui + super(Elasticsearch2BoostBackendTestCase, self).tearDown() + + def raw_search(self, query): + return self.raw_es.search(q='*:*', index=settings.HAYSTACK_CONNECTIONS['elasticsearch']['INDEX_NAME']) + + def test_boost(self): + self.sb.update(self.smmi, self.sample_objs) + self.assertEqual(self.raw_search('*:*')['hits']['total'], 4) + + results = SearchQuerySet(using='elasticsearch').filter(SQ(author='daniel') | SQ(editor='daniel')) + + self.assertEqual(set([result.id for result in results]), + {'core.afourthmockmodel.4', 'core.afourthmockmodel.3', 'core.afourthmockmodel.1', + 'core.afourthmockmodel.2'}) + + def test__to_python(self): + self.assertEqual(self.sb._to_python('abc'), 'abc') + self.assertEqual(self.sb._to_python('1'), 1) + self.assertEqual(self.sb._to_python('2653'), 2653) + self.assertEqual(self.sb._to_python('25.5'), 25.5) + self.assertEqual(self.sb._to_python('[1, 2, 3]'), [1, 2, 3]) + self.assertEqual(self.sb._to_python('{"a": 1, "b": 2, "c": 3}'), {'a': 1, 'c': 3, 'b': 2}) + self.assertEqual(self.sb._to_python('2009-05-09T16:14:00'), datetime.datetime(2009, 5, 9, 16, 14)) + self.assertEqual(self.sb._to_python('2009-05-09T00:00:00'), datetime.datetime(2009, 5, 9, 0, 0)) + self.assertEqual(self.sb._to_python(None), None) + + +class RecreateIndexTestCase(TestCase): + def setUp(self): + self.raw_es = elasticsearch.Elasticsearch( + settings.HAYSTACK_CONNECTIONS['elasticsearch']['URL']) + + def test_recreate_index(self): + clear_elasticsearch_index() + + sb = connections['elasticsearch'].get_backend() + sb.silently_fail = True + sb.setup() + + original_mapping = self.raw_es.indices.get_mapping(index=sb.index_name) + + sb.clear() + sb.setup() + + try: + updated_mapping = self.raw_es.indices.get_mapping(sb.index_name) + except elasticsearch.NotFoundError: + self.fail("There is no mapping after recreating the index") + + self.assertEqual(original_mapping, updated_mapping, + "Mapping after recreating the index differs from the original one") + + +class Elasticsearch2FacetingTestCase(TestCase): + def setUp(self): + super(Elasticsearch2FacetingTestCase, self).setUp() + + # Wipe it clean. + clear_elasticsearch_index() + + # Stow. + self.old_ui = connections['elasticsearch'].get_unified_index() + self.ui = UnifiedIndex() + self.smmi = Elasticsearch2FacetingMockSearchIndex() + self.ui.build(indexes=[self.smmi]) + connections['elasticsearch']._index = self.ui + self.sb = connections['elasticsearch'].get_backend() + + # Force the backend to rebuild the mapping each time. + self.sb.existing_mapping = {} + self.sb.setup() + + self.sample_objs = [] + + for i in range(1, 10): + mock = AFourthMockModel() + mock.id = i + if i > 5: + mock.editor = 'George Taylor' + else: + mock.editor = 'Perry White' + if i % 2: + mock.author = 'Daniel Lindsley' + else: + mock.author = 'Dan Watson' + mock.pub_date = datetime.date(2013, 9, (i % 4) + 1) + self.sample_objs.append(mock) + + def tearDown(self): + connections['elasticsearch']._index = self.old_ui + super(Elasticsearch2FacetingTestCase, self).tearDown() + + def test_facet(self): + self.sb.update(self.smmi, self.sample_objs) + counts = SearchQuerySet('elasticsearch').facet('author').facet('editor').facet_counts() + self.assertEqual(counts['fields']['author'], [ + ('Daniel Lindsley', 5), + ('Dan Watson', 4), + ]) + self.assertEqual(counts['fields']['editor'], [ + ('Perry White', 5), + ('George Taylor', 4), + ]) + counts = SearchQuerySet('elasticsearch').filter(content='white').facet('facet_field', + order='reverse_count').facet_counts() + self.assertEqual(counts['fields']['facet_field'], [ + ('Dan Watson', 2), + ('Daniel Lindsley', 3), + ]) + + def test_multiple_narrow(self): + self.sb.update(self.smmi, self.sample_objs) + counts = SearchQuerySet('elasticsearch').narrow('editor_exact:"Perry White"').narrow( + 'author_exact:"Daniel Lindsley"').facet('author').facet_counts() + self.assertEqual(counts['fields']['author'], [ + ('Daniel Lindsley', 3), + ]) + + def test_narrow(self): + self.sb.update(self.smmi, self.sample_objs) + counts = SearchQuerySet('elasticsearch').facet('author').facet('editor').narrow( + 'editor_exact:"Perry White"').facet_counts() + self.assertEqual(counts['fields']['author'], [ + ('Daniel Lindsley', 3), + ('Dan Watson', 2), + ]) + self.assertEqual(counts['fields']['editor'], [ + ('Perry White', 5), + ]) + + def test_date_facet(self): + self.sb.update(self.smmi, self.sample_objs) + start = datetime.date(2013, 9, 1) + end = datetime.date(2013, 9, 30) + # Facet by day + counts = SearchQuerySet('elasticsearch').date_facet('pub_date', start_date=start, end_date=end, + gap_by='day').facet_counts() + self.assertEqual(counts['dates']['pub_date'], [ + (datetime.datetime(2013, 9, 1), 2), + (datetime.datetime(2013, 9, 2), 3), + (datetime.datetime(2013, 9, 3), 2), + (datetime.datetime(2013, 9, 4), 2), + ]) + # By month + counts = SearchQuerySet('elasticsearch').date_facet('pub_date', start_date=start, end_date=end, + gap_by='month').facet_counts() + self.assertEqual(counts['dates']['pub_date'], [ + (datetime.datetime(2013, 9, 1), 9), + ]) diff --git a/test_haystack/elasticsearch2_tests/test_inputs.py b/test_haystack/elasticsearch2_tests/test_inputs.py new file mode 100644 index 000000000..adc87d16d --- /dev/null +++ b/test_haystack/elasticsearch2_tests/test_inputs.py @@ -0,0 +1,85 @@ +# -*- coding: utf-8 -*- + +from __future__ import absolute_import, division, print_function, unicode_literals + +from django.test import TestCase + +from haystack import connections, inputs + + +class Elasticsearch2InputTestCase(TestCase): + def setUp(self): + super(Elasticsearch2InputTestCase, self).setUp() + self.query_obj = connections['elasticsearch'].get_query() + + def test_raw_init(self): + raw = inputs.Raw('hello OR there, :you') + self.assertEqual(raw.query_string, 'hello OR there, :you') + self.assertEqual(raw.kwargs, {}) + self.assertEqual(raw.post_process, False) + + raw = inputs.Raw('hello OR there, :you', test='really') + self.assertEqual(raw.query_string, 'hello OR there, :you') + self.assertEqual(raw.kwargs, {'test': 'really'}) + self.assertEqual(raw.post_process, False) + + def test_raw_prepare(self): + raw = inputs.Raw('hello OR there, :you') + self.assertEqual(raw.prepare(self.query_obj), 'hello OR there, :you') + + def test_clean_init(self): + clean = inputs.Clean('hello OR there, :you') + self.assertEqual(clean.query_string, 'hello OR there, :you') + self.assertEqual(clean.post_process, True) + + def test_clean_prepare(self): + clean = inputs.Clean('hello OR there, :you') + self.assertEqual(clean.prepare(self.query_obj), 'hello or there, \\:you') + + def test_exact_init(self): + exact = inputs.Exact('hello OR there, :you') + self.assertEqual(exact.query_string, 'hello OR there, :you') + self.assertEqual(exact.post_process, True) + + def test_exact_prepare(self): + exact = inputs.Exact('hello OR there, :you') + self.assertEqual(exact.prepare(self.query_obj), u'"hello OR there, :you"') + + exact = inputs.Exact('hello OR there, :you', clean=True) + self.assertEqual(exact.prepare(self.query_obj), u'"hello or there, \\:you"') + + def test_not_init(self): + not_it = inputs.Not('hello OR there, :you') + self.assertEqual(not_it.query_string, 'hello OR there, :you') + self.assertEqual(not_it.post_process, True) + + def test_not_prepare(self): + not_it = inputs.Not('hello OR there, :you') + self.assertEqual(not_it.prepare(self.query_obj), u'NOT (hello or there, \\:you)') + + def test_autoquery_init(self): + autoquery = inputs.AutoQuery('panic -don\'t "froody dude"') + self.assertEqual(autoquery.query_string, 'panic -don\'t "froody dude"') + self.assertEqual(autoquery.post_process, False) + + def test_autoquery_prepare(self): + autoquery = inputs.AutoQuery('panic -don\'t "froody dude"') + self.assertEqual(autoquery.prepare(self.query_obj), u'panic NOT don\'t "froody dude"') + + def test_altparser_init(self): + altparser = inputs.AltParser('dismax') + self.assertEqual(altparser.parser_name, 'dismax') + self.assertEqual(altparser.query_string, '') + self.assertEqual(altparser.kwargs, {}) + self.assertEqual(altparser.post_process, False) + + altparser = inputs.AltParser('dismax', 'douglas adams', qf='author', mm=1) + self.assertEqual(altparser.parser_name, 'dismax') + self.assertEqual(altparser.query_string, 'douglas adams') + self.assertEqual(altparser.kwargs, {'mm': 1, 'qf': 'author'}) + self.assertEqual(altparser.post_process, False) + + def test_altparser_prepare(self): + altparser = inputs.AltParser('dismax', 'douglas adams', qf='author', mm=1) + self.assertEqual(altparser.prepare(self.query_obj), + u"""{!dismax mm=1 qf=author v='douglas adams'}""") diff --git a/test_haystack/elasticsearch2_tests/test_query.py b/test_haystack/elasticsearch2_tests/test_query.py new file mode 100644 index 000000000..c66191c59 --- /dev/null +++ b/test_haystack/elasticsearch2_tests/test_query.py @@ -0,0 +1,209 @@ +# -*- coding: utf-8 -*- +from __future__ import absolute_import, division, print_function, unicode_literals + +import datetime + +import elasticsearch +from django.test import TestCase + +from haystack import connections +from haystack.inputs import Exact +from haystack.models import SearchResult +from haystack.query import SearchQuerySet, SQ +from haystack.utils.geo import D, Point +from ..core.models import AnotherMockModel, MockModel + + +class Elasticsearch2SearchQueryTestCase(TestCase): + def setUp(self): + super(Elasticsearch2SearchQueryTestCase, self).setUp() + self.sq = connections['elasticsearch'].get_query() + + def test_build_query_all(self): + self.assertEqual(self.sq.build_query(), '*:*') + + def test_build_query_single_word(self): + self.sq.add_filter(SQ(content='hello')) + self.assertEqual(self.sq.build_query(), '(hello)') + + def test_build_query_boolean(self): + self.sq.add_filter(SQ(content=True)) + self.assertEqual(self.sq.build_query(), '(True)') + + def test_regression_slash_search(self): + self.sq.add_filter(SQ(content='hello/')) + self.assertEqual(self.sq.build_query(), '(hello\\/)') + + def test_build_query_datetime(self): + self.sq.add_filter(SQ(content=datetime.datetime(2009, 5, 8, 11, 28))) + self.assertEqual(self.sq.build_query(), '(2009-05-08T11:28:00)') + + def test_build_query_multiple_words_and(self): + self.sq.add_filter(SQ(content='hello')) + self.sq.add_filter(SQ(content='world')) + self.assertEqual(self.sq.build_query(), '((hello) AND (world))') + + def test_build_query_multiple_words_not(self): + self.sq.add_filter(~SQ(content='hello')) + self.sq.add_filter(~SQ(content='world')) + self.assertEqual(self.sq.build_query(), '(NOT ((hello)) AND NOT ((world)))') + + def test_build_query_multiple_words_or(self): + self.sq.add_filter(~SQ(content='hello')) + self.sq.add_filter(SQ(content='hello'), use_or=True) + self.assertEqual(self.sq.build_query(), '(NOT ((hello)) OR (hello))') + + def test_build_query_multiple_words_mixed(self): + self.sq.add_filter(SQ(content='why')) + self.sq.add_filter(SQ(content='hello'), use_or=True) + self.sq.add_filter(~SQ(content='world')) + self.assertEqual(self.sq.build_query(), u'(((why) OR (hello)) AND NOT ((world)))') + + def test_build_query_phrase(self): + self.sq.add_filter(SQ(content='hello world')) + self.assertEqual(self.sq.build_query(), '(hello AND world)') + + self.sq.add_filter(SQ(content__exact='hello world')) + self.assertEqual(self.sq.build_query(), u'((hello AND world) AND ("hello world"))') + + def test_build_query_boost(self): + self.sq.add_filter(SQ(content='hello')) + self.sq.add_boost('world', 5) + self.assertEqual(self.sq.build_query(), "(hello) world^5") + + def test_build_query_multiple_filter_types(self): + self.sq.add_filter(SQ(content='why')) + self.sq.add_filter(SQ(pub_date__lte=Exact('2009-02-10 01:59:00'))) + self.sq.add_filter(SQ(author__gt='daniel')) + self.sq.add_filter(SQ(created__lt=Exact('2009-02-12 12:13:00'))) + self.sq.add_filter(SQ(title__gte='B')) + self.sq.add_filter(SQ(id__in=[1, 2, 3])) + self.sq.add_filter(SQ(rating__range=[3, 5])) + self.assertEqual(self.sq.build_query(), + u'((why) AND pub_date:([* TO "2009-02-10 01:59:00"]) AND author:({"daniel" TO *}) AND created:({* TO "2009-02-12 12:13:00"}) AND title:(["B" TO *]) AND id:("1" OR "2" OR "3") AND rating:(["3" TO "5"]))') + + def test_build_query_multiple_filter_types_with_datetimes(self): + self.sq.add_filter(SQ(content='why')) + self.sq.add_filter(SQ(pub_date__lte=datetime.datetime(2009, 2, 10, 1, 59, 0))) + self.sq.add_filter(SQ(author__gt='daniel')) + self.sq.add_filter(SQ(created__lt=datetime.datetime(2009, 2, 12, 12, 13, 0))) + self.sq.add_filter(SQ(title__gte='B')) + self.sq.add_filter(SQ(id__in=[1, 2, 3])) + self.sq.add_filter(SQ(rating__range=[3, 5])) + self.assertEqual(self.sq.build_query(), + u'((why) AND pub_date:([* TO "2009-02-10T01:59:00"]) AND author:({"daniel" TO *}) AND created:({* TO "2009-02-12T12:13:00"}) AND title:(["B" TO *]) AND id:("1" OR "2" OR "3") AND rating:(["3" TO "5"]))') + + def test_build_query_in_filter_multiple_words(self): + self.sq.add_filter(SQ(content='why')) + self.sq.add_filter(SQ(title__in=["A Famous Paper", "An Infamous Article"])) + self.assertEqual(self.sq.build_query(), u'((why) AND title:("A Famous Paper" OR "An Infamous Article"))') + + def test_build_query_in_filter_datetime(self): + self.sq.add_filter(SQ(content='why')) + self.sq.add_filter(SQ(pub_date__in=[datetime.datetime(2009, 7, 6, 1, 56, 21)])) + self.assertEqual(self.sq.build_query(), u'((why) AND pub_date:("2009-07-06T01:56:21"))') + + def test_build_query_in_with_set(self): + self.sq.add_filter(SQ(content='why')) + self.sq.add_filter(SQ(title__in={"A Famous Paper", "An Infamous Article"})) + self.assertTrue('((why) AND title:(' in self.sq.build_query()) + self.assertTrue('"A Famous Paper"' in self.sq.build_query()) + self.assertTrue('"An Infamous Article"' in self.sq.build_query()) + + def test_build_query_wildcard_filter_types(self): + self.sq.add_filter(SQ(content='why')) + self.sq.add_filter(SQ(title__startswith='haystack')) + self.assertEqual(self.sq.build_query(), u'((why) AND title:(haystack*))') + + def test_build_query_fuzzy_filter_types(self): + self.sq.add_filter(SQ(content='why')) + self.sq.add_filter(SQ(title__fuzzy='haystack')) + self.assertEqual(self.sq.build_query(), u'((why) AND title:(haystack~))') + + def test_clean(self): + self.assertEqual(self.sq.clean('hello world'), 'hello world') + self.assertEqual(self.sq.clean('hello AND world'), 'hello and world') + self.assertEqual(self.sq.clean('hello AND OR NOT TO + - && || ! ( ) { } [ ] ^ " ~ * ? : \ / world'), + 'hello and or not to \\+ \\- \\&& \\|| \\! \\( \\) \\{ \\} \\[ \\] \\^ \\" \\~ \\* \\? \\: \\\\ \\/ world') + self.assertEqual(self.sq.clean('so please NOTe i am in a bAND and bORed'), + 'so please NOTe i am in a bAND and bORed') + + def test_build_query_with_models(self): + self.sq.add_filter(SQ(content='hello')) + self.sq.add_model(MockModel) + self.assertEqual(self.sq.build_query(), '(hello)') + + self.sq.add_model(AnotherMockModel) + self.assertEqual(self.sq.build_query(), u'(hello)') + + def test_set_result_class(self): + # Assert that we're defaulting to ``SearchResult``. + self.assertTrue(issubclass(self.sq.result_class, SearchResult)) + + # Custom class. + class IttyBittyResult(object): + pass + + self.sq.set_result_class(IttyBittyResult) + self.assertTrue(issubclass(self.sq.result_class, IttyBittyResult)) + + # Reset to default. + self.sq.set_result_class(None) + self.assertTrue(issubclass(self.sq.result_class, SearchResult)) + + def test_in_filter_values_list(self): + self.sq.add_filter(SQ(content='why')) + self.sq.add_filter(SQ(title__in=[1, 2, 3])) + self.assertEqual(self.sq.build_query(), u'((why) AND title:("1" OR "2" OR "3"))') + + def test_narrow_sq(self): + sqs = SearchQuerySet(using='elasticsearch').narrow(SQ(foo='moof')) + self.assertTrue(isinstance(sqs, SearchQuerySet)) + self.assertEqual(len(sqs.query.narrow_queries), 1) + self.assertEqual(sqs.query.narrow_queries.pop(), 'foo:(moof)') + + +class Elasticsearch2SearchQuerySpatialBeforeReleaseTestCase(TestCase): + def setUp(self): + super(Elasticsearch2SearchQuerySpatialBeforeReleaseTestCase, self).setUp() + self.backend = connections['elasticsearch'].get_backend() + self._elasticsearch_version = elasticsearch.VERSION + elasticsearch.VERSION = (0, 9, 9) + + def tearDown(self): + elasticsearch.VERSION = self._elasticsearch_version + + def test_build_query_with_dwithin_range(self): + """ + Test build_search_kwargs with dwithin range for Elasticsearch versions < 1.0.0 + """ + search_kwargs = self.backend.build_search_kwargs('where', dwithin={ + 'field': "location_field", + 'point': Point(1.2345678, 2.3456789), + 'distance': D(m=500) + }) + self.assertEqual(search_kwargs['query']['filtered']['filter']['bool']['must'][1]['geo_distance'], + {'distance': 0.5, 'location_field': {'lat': 2.3456789, 'lon': 1.2345678}}) + + +class Elasticsearch2SearchQuerySpatialAfterReleaseTestCase(TestCase): + def setUp(self): + super(Elasticsearch2SearchQuerySpatialAfterReleaseTestCase, self).setUp() + self.backend = connections['elasticsearch'].get_backend() + self._elasticsearch_version = elasticsearch.VERSION + elasticsearch.VERSION = (1, 0, 0) + + def tearDown(self): + elasticsearch.VERSION = self._elasticsearch_version + + def test_build_query_with_dwithin_range(self): + """ + Test build_search_kwargs with dwithin range for Elasticsearch versions >= 1.0.0 + """ + search_kwargs = self.backend.build_search_kwargs('where', dwithin={ + 'field': "location_field", + 'point': Point(1.2345678, 2.3456789), + 'distance': D(m=500) + }) + self.assertEqual(search_kwargs['query']['filtered']['filter']['bool']['must'][1]['geo_distance'], + {'distance': "0.500000km", 'location_field': {'lat': 2.3456789, 'lon': 1.2345678}}) diff --git a/test_haystack/elasticsearch_tests/__init__.py b/test_haystack/elasticsearch_tests/__init__.py index 4066af099..121fe22c7 100644 --- a/test_haystack/elasticsearch_tests/__init__.py +++ b/test_haystack/elasticsearch_tests/__init__.py @@ -10,6 +10,9 @@ def setup(): try: + import elasticsearch + if not ((1, 0, 0) <= elasticsearch.__version__ < (2, 0, 0)): + raise ImportError from elasticsearch import Elasticsearch, ElasticsearchException except ImportError: raise unittest.SkipTest("elasticsearch-py not installed.") @@ -18,5 +21,5 @@ def setup(): try: es.info() except ElasticsearchException as e: - raise unittest.SkipTest("elasticsearch not running on %r" % settings.HAYSTACK_CONNECTIONS['elasticsearch']['URL'], e) - + raise unittest.SkipTest("elasticsearch not running on %r" % \ + settings.HAYSTACK_CONNECTIONS['elasticsearch']['URL'], e) diff --git a/test_haystack/settings.py b/test_haystack/settings.py index 43bf75f0d..55e11bbd2 100644 --- a/test_haystack/settings.py +++ b/test_haystack/settings.py @@ -93,3 +93,11 @@ 'INCLUDE_SPELLING': True, }, } + +if os.getenv('VERSION_ES') == ">=2.0.0,<3.0.0": + HAYSTACK_CONNECTIONS['elasticsearch'] = { + 'ENGINE': 'haystack.backends.elasticsearch2_backend.Elasticsearch2SearchEngine', + 'URL': '127.0.0.1:9200/', + 'INDEX_NAME': 'test_default', + 'INCLUDE_SPELLING': True, + } From 0272b757d1487f151e804cf8fba0cb24e97ac03b Mon Sep 17 00:00:00 2001 From: joaojunior Date: Tue, 28 Jun 2016 21:19:19 +0000 Subject: [PATCH 02/20] Update Files to run tests in Elasticsearch2.x --- .travis.yml | 22 +++++-- setup.py | 1 - tox.ini | 184 +++++++++++++++++++++++++++++++++++++++++++++------- 3 files changed, 178 insertions(+), 29 deletions(-) diff --git a/.travis.yml b/.travis.yml index 48dd98376..e4445cdd1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,10 +26,21 @@ addons: before_install: - mkdir -p $HOME/download-cache + - > + if [[ $VERSION_ES == '>=2.0.0,<3.0.0' ]]; + then + wget https://download.elasticsearch.org/elasticsearch/release/org/elasticsearch/distribution/tar/elasticsearch/2.2.1/elasticsearch-2.2.1.tar.gz + tar zxf elasticsearch-2.2.1.tar.gz + elasticsearch-2.2.1/bin/elasticsearch -d -Dhttp.port=9200 + else + wget https://download.elastic.co/elasticsearch/elasticsearch/elasticsearch-1.7.5.tar.gz + tar zxf elasticsearch-1.7.5.tar.gz + elasticsearch-1.7.5/bin/elasticsearch -d -Dhttp.port=9200 + fi install: - pip install --upgrade setuptools - - pip install requests "Django${DJANGO_VERSION}" + - pip install requests "Django${DJANGO_VERSION}" "elasticsearch${VERSION_ES}" - python setup.py clean build install before_script: @@ -41,9 +52,12 @@ script: env: matrix: - - DJANGO_VERSION=">=1.8,<1.9" - - DJANGO_VERSION=">=1.9,<1.10" - - DJANGO_VERSION=">=1.10,<1.11" + - DJANGO_VERSION=">=1.8,<1.9" VERSION_ES=">=1.0.0,<2.0.0" + - DJANGO_VERSION=">=1.9,<1.10" VERSION_ES=">=1.0.0,<2.0.0" + - DJANGO_VERSION=">=1.10,<1.11" VERSION_ES=">=1.0.0,<2.0.0" + - DJANGO_VERSION=">=1.8,<1.9" VERSION_ES=">=2.0.0,<3.0.0" + - DJANGO_VERSION=">=1.9,<1.10" VERSION_ES=">=2.0.0,<3.0.0" + - DJANGO_VERSION=">=1.10,<1.11" VERSION_ES=">=2.0.0,<3.0.0" matrix: allow_failures: diff --git a/setup.py b/setup.py index 1e54677d1..496703630 100755 --- a/setup.py +++ b/setup.py @@ -17,7 +17,6 @@ ] tests_require = [ - 'elasticsearch>=1.0.0,<2.0.0', 'pysolr>=3.3.2', 'whoosh>=2.5.4,<3.0', 'python-dateutil', diff --git a/tox.ini b/tox.ini index b44f2ee84..1bbea64fa 100644 --- a/tox.ini +++ b/tox.ini @@ -1,17 +1,29 @@ [tox] envlist = docs, - py27-django1.8, - py27-django1.9, - py27-django1.10, - py34-django1.8, - py34-django1.9, - py34-django1.10, - py35-django1.8, - py35-django1.9, - py35-django1.10, - pypy-django1.8, - pypy-django1.9, - pypy-django1.10, + py27-django1.8-es1.x, + py27-django1.9-es1.x, + py27-django1.10-es1.x, + py34-django1.8-es1.x, + py34-django1.9-es1.x, + py34-django1.10-es1.x, + py35-django1.8-es1.x, + py35-django1.9-es1.x, + py35-django1.10-es1.x, + pypy-django1.8-es1.x, + pypy-django1.9-es1.x, + pypy-django1.10-es1.x, + py27-django1.8-es2.x, + py27-django1.9-es2.x, + py27-django1.10-es2.x, + py34-django1.8-es2.x, + py34-django1.9-es2.x, + py34-django1.10-es2.x, + py35-django1.8-es2.x, + py35-django1.9-es2.x, + py35-django1.10-es2.x, + pypy-django1.8-es2.x, + pypy-django1.9-es2.x, + pypy-django1.10-es2.x, [base] deps = requests @@ -28,77 +40,201 @@ deps = deps = Django>=1.8,<1.9 +[es2.x] +deps = + elasticsearch>=2.0.0,<3.0.0 + +[es1.x] +deps = + elasticsearch>=1.0.0,<2.0.0 + [testenv] commands = python test_haystack/solr_tests/server/wait-for-solr python {toxinidir}/setup.py test -[testenv:pypy-django1.8] +[testenv:pypy-django1.8-es1.x] +setenv = VERSION_ES=>=1.0.0,<2.0.0 +deps = + {[es1.x]deps} + {[django1.8]deps} + {[base]deps} + +[testenv:pypy-django1.9-es1.x] +setenv = VERSION_ES=>=1.0.0,<2.0.0 +deps = + {[es1.x]deps} + {[django1.9]deps} + {[base]deps} + +[testenv:pypy-django1.10-es1.x] +setenv = VERSION_ES=>=1.0.0,<2.0.0 +deps = + {[es1.x]deps} + {[django1.10]deps} + {[base]deps} + +[testenv:py27-django1.8-es1.x] +basepython = python2.7 +setenv = VERSION_ES=>=1.0.0,<2.0.0 +deps = + {[es1.x]deps} + {[django1.8]deps} + {[base]deps} + +[testenv:py27-django1.9-es1.x] +basepython = python2.7 +setenv = VERSION_ES=>=1.0.0,<2.0.0 +deps = + {[es1.x]deps} + {[django1.9]deps} + {[base]deps} + +[testenv:py27-django1.10-es1.x] +basepython = python2.7 +setenv = VERSION_ES=>=1.0.0,<2.0.0 +deps = + {[es1.x]deps} + {[django1.10]deps} + {[base]deps} + +[testenv:py34-django1.8-es1.x] +basepython = python3.4 +setenv = VERSION_ES=>=1.0.0,<2.0.0 +deps = + {[es1.x]deps} + {[django1.8]deps} + {[base]deps} + +[testenv:py34-django1.9-es1.x] +basepython = python3.4 +setenv = VERSION_ES=>=1.0.0,<2.0.0 +deps = + {[es1.x]deps} + {[django1.9]deps} + {[base]deps} + +[testenv:py34-django1.10-es1.x] +basepython = python3.4 +setenv = VERSION_ES=>=1.0.0,<2.0.0 +deps = + {[django1.10]deps} + {[base]deps} + +[testenv:py35-django1.8-es1.x] +basepython = python3.5 +setenv = VERSION_ES=>=1.0.0,<2.0.0 +deps = + {[es1.x]deps} + {[django1.8]deps} + {[base]deps} + +[testenv:py35-django1.9-es1.x] +basepython = python3.5 +setenv = VERSION_ES=>=1.0.0,<2.0.0 +deps = + {[es1.x]deps} + {[django1.9]deps} + {[base]deps} + +[testenv:py35-django1.10-es1.x] +basepython = python3.5 +setenv = VERSION_ES=>=1.0.0,<2.0.0 +deps = + {[es1.x]deps} + {[django1.10]deps} + {[base]deps} + +[testenv:pypy-django1.8-es2.x] +setenv = VERSION_ES=>=2.0.0,<3.0.0 deps = + {[es2.x]deps} {[django1.8]deps} {[base]deps} -[testenv:pypy-django1.9] +[testenv:pypy-django1.9-es2.x] +setenv = VERSION_ES=>=2.0.0,<3.0.0 deps = + {[es2.x]deps} {[django1.9]deps} {[base]deps} -[testenv:pypy-django1.10] +[testenv:pypy-django1.10-es2.x] +setenv = VERSION_ES=>=2.0.0,<3.0.0 deps = + {[es2.x]deps} {[django1.10]deps} {[base]deps} -[testenv:py27-django1.8] +[testenv:py27-django1.8-es2.x] basepython = python2.7 +setenv = VERSION_ES=>=2.0.0,<3.0.0 deps = + {[es2.x]deps} {[django1.8]deps} {[base]deps} -[testenv:py27-django1.9] +[testenv:py27-django1.9-es2.x] basepython = python2.7 +setenv = VERSION_ES=>=2.0.0,<3.0.0 deps = + {[es2.x]deps} {[django1.9]deps} {[base]deps} -[testenv:py27-django1.10] +[testenv:py27-django1.10-es2.x] basepython = python2.7 +setenv = VERSION_ES=>=2.0.0,<3.0.0 deps = + {[es2.x]deps} {[django1.10]deps} {[base]deps} -[testenv:py34-django1.8] +[testenv:py34-django1.8-es2.x] basepython = python3.4 +setenv = VERSION_ES=>=2.0.0,<3.0.0 deps = + {[es2.x]deps} {[django1.8]deps} {[base]deps} -[testenv:py34-django1.9] +[testenv:py34-django1.9-es2.x] basepython = python3.4 +setenv = VERSION_ES=>=2.0.0,<3.0.0 deps = + {[es2.x]deps} {[django1.9]deps} {[base]deps} -[testenv:py34-django1.10] +[testenv:py34-django1.10-es2.x] basepython = python3.4 +setenv = VERSION_ES=>=2.0.0,<3.0.0 deps = + {[es2.x]deps} {[django1.10]deps} {[base]deps} -[testenv:py35-django1.8] +[testenv:py35-django1.8-es2.x] basepython = python3.5 +setenv = VERSION_ES=>=2.0.0,<3.0.0 deps = + {[es2.x]deps} {[django1.8]deps} {[base]deps} -[testenv:py35-django1.9] +[testenv:py35-django1.9-es2.x] basepython = python3.5 +setenv = VERSION_ES=>=2.0.0,<3.0.0 deps = + {[es2.x]deps} {[django1.9]deps} {[base]deps} -[testenv:py35-django1.10] +[testenv:py35-django1.10-es2.x] basepython = python3.5 +setenv = VERSION_ES=>=2.0.0,<3.0.0 deps = + {[es2.x]deps} {[django1.10]deps} {[base]deps} From 7fc63886d12087e19371a339a08360880c11f16c Mon Sep 17 00:00:00 2001 From: joaojunior Date: Wed, 13 Jul 2016 21:55:23 +0000 Subject: [PATCH 03/20] Install elasticsearch1.7 via apt --- .travis.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index e4445cdd1..6a9b94366 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,9 +33,11 @@ before_install: tar zxf elasticsearch-2.2.1.tar.gz elasticsearch-2.2.1/bin/elasticsearch -d -Dhttp.port=9200 else - wget https://download.elastic.co/elasticsearch/elasticsearch/elasticsearch-1.7.5.tar.gz - tar zxf elasticsearch-1.7.5.tar.gz - elasticsearch-1.7.5/bin/elasticsearch -d -Dhttp.port=9200 + wget -qO - https://packages.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add - + echo "deb http://packages.elastic.co/elasticsearch/1.7/debian stable main" | sudo tee -a /etc/apt/sources.list.d/elasticsearch-1.7.list + sudo apt-get update + sudo apt-get -y install elasticsearch + sudo service elasticsearch restart fi install: From 7563ce720335331b298f35fe540f6a2bc627890e Mon Sep 17 00:00:00 2001 From: joaojunior Date: Wed, 13 Jul 2016 22:03:55 +0000 Subject: [PATCH 04/20] Get changes from Master to resolve conflicts --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 6a9b94366..a80c76155 100644 --- a/.travis.yml +++ b/.travis.yml @@ -67,6 +67,7 @@ matrix: services: - elasticsearch + - python: "pypy" notifications: irc: "irc.freenode.org#haystack" From c9dc8603a56fd263631ee6237286641aaa932209 Mon Sep 17 00:00:00 2001 From: joaojunior Date: Thu, 14 Jul 2016 00:16:08 +0000 Subject: [PATCH 05/20] Add logging in __init__ tests elasticsearch --- test_haystack/elasticsearch2_tests/__init__.py | 4 ++++ test_haystack/elasticsearch_tests/__init__.py | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/test_haystack/elasticsearch2_tests/__init__.py b/test_haystack/elasticsearch2_tests/__init__.py index ba6384f46..326960ca0 100644 --- a/test_haystack/elasticsearch2_tests/__init__.py +++ b/test_haystack/elasticsearch2_tests/__init__.py @@ -4,17 +4,20 @@ from django.conf import settings from ..utils import unittest +from haystack.utils import log as logging warnings.simplefilter('ignore', Warning) def setup(): + log = logging.getLogger('haystack') try: import elasticsearch if not ((2, 0, 0) <= elasticsearch.__version__ < (3, 0, 0)): raise ImportError from elasticsearch import Elasticsearch, exceptions except ImportError: + self.log.error("'elasticsearch>=2.0.0,<3.0.0' not installed.", exc_info=True) raise unittest.SkipTest("'elasticsearch>=2.0.0,<3.0.0' not installed.") url = settings.HAYSTACK_CONNECTIONS['elasticsearch']['URL'] @@ -22,4 +25,5 @@ def setup(): try: es.info() except exceptions.ConnectionError as e: + self.log.error("elasticsearch not running on %r" % url, exc_info=True) raise unittest.SkipTest("elasticsearch not running on %r" % url, e) diff --git a/test_haystack/elasticsearch_tests/__init__.py b/test_haystack/elasticsearch_tests/__init__.py index 121fe22c7..b57bcebfb 100644 --- a/test_haystack/elasticsearch_tests/__init__.py +++ b/test_haystack/elasticsearch_tests/__init__.py @@ -5,21 +5,26 @@ from django.conf import settings +from haystack.utils import log as logging warnings.simplefilter('ignore', Warning) def setup(): + log = logging.getLogger('haystack') try: import elasticsearch if not ((1, 0, 0) <= elasticsearch.__version__ < (2, 0, 0)): raise ImportError from elasticsearch import Elasticsearch, ElasticsearchException except ImportError: + self.log.error("elasticsearch-py not installed.", exc_info=True) raise unittest.SkipTest("elasticsearch-py not installed.") es = Elasticsearch(settings.HAYSTACK_CONNECTIONS['elasticsearch']['URL']) try: es.info() except ElasticsearchException as e: + self.log.error("elasticsearch not running on %r" % \ + settings.HAYSTACK_CONNECTIONS['elasticsearch']['URL'], exc_info=True) raise unittest.SkipTest("elasticsearch not running on %r" % \ settings.HAYSTACK_CONNECTIONS['elasticsearch']['URL'], e) From 895b559479b31e2eae2d6ac739ac75b3446fc26c Mon Sep 17 00:00:00 2001 From: joaojunior Date: Fri, 15 Jul 2016 00:34:43 +0000 Subject: [PATCH 06/20] Fix .travis --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a80c76155..6a9b94366 100644 --- a/.travis.yml +++ b/.travis.yml @@ -67,7 +67,6 @@ matrix: services: - elasticsearch - - python: "pypy" notifications: irc: "irc.freenode.org#haystack" From e857a2d3e37e943bffb3785cd02265687955c9ef Mon Sep 17 00:00:00 2001 From: joaojunior Date: Fri, 15 Jul 2016 23:42:46 +0000 Subject: [PATCH 07/20] sudo=true in .travis.yml to install elasticsearch from apt-get --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 6a9b94366..511305034 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,4 @@ -sudo: false +sudo: true language: python python: - 2.7 From 34b2e37ccda443046e0b5b6a8273b36fab37d97c Mon Sep 17 00:00:00 2001 From: joaojunior Date: Fri, 15 Jul 2016 23:59:03 +0000 Subject: [PATCH 08/20] Fix typo --- test_haystack/elasticsearch2_tests/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test_haystack/elasticsearch2_tests/__init__.py b/test_haystack/elasticsearch2_tests/__init__.py index 326960ca0..b8f51fed2 100644 --- a/test_haystack/elasticsearch2_tests/__init__.py +++ b/test_haystack/elasticsearch2_tests/__init__.py @@ -17,7 +17,7 @@ def setup(): raise ImportError from elasticsearch import Elasticsearch, exceptions except ImportError: - self.log.error("'elasticsearch>=2.0.0,<3.0.0' not installed.", exc_info=True) + log.error("'elasticsearch>=2.0.0,<3.0.0' not installed.", exc_info=True) raise unittest.SkipTest("'elasticsearch>=2.0.0,<3.0.0' not installed.") url = settings.HAYSTACK_CONNECTIONS['elasticsearch']['URL'] @@ -25,5 +25,5 @@ def setup(): try: es.info() except exceptions.ConnectionError as e: - self.log.error("elasticsearch not running on %r" % url, exc_info=True) + log.error("elasticsearch not running on %r" % url, exc_info=True) raise unittest.SkipTest("elasticsearch not running on %r" % url, e) From 8e90f75894d4f7bc584b6cfbfdbce868765e0e06 Mon Sep 17 00:00:00 2001 From: joaojunior Date: Sun, 24 Jul 2016 18:55:35 +0000 Subject: [PATCH 09/20] Remove services elasticsearch --- .travis.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 511305034..1c007aab1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -65,9 +65,6 @@ matrix: allow_failures: - python: 'pypy' -services: - - elasticsearch - notifications: irc: "irc.freenode.org#haystack" email: false From a2ec3a4eb27592d05356b1b534ef7b61cde1d591 Mon Sep 17 00:00:00 2001 From: joaojunior Date: Sun, 24 Jul 2016 18:56:47 +0000 Subject: [PATCH 10/20] Remove typo --- test_haystack/elasticsearch_tests/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test_haystack/elasticsearch_tests/__init__.py b/test_haystack/elasticsearch_tests/__init__.py index b57bcebfb..67dda6de6 100644 --- a/test_haystack/elasticsearch_tests/__init__.py +++ b/test_haystack/elasticsearch_tests/__init__.py @@ -17,14 +17,14 @@ def setup(): raise ImportError from elasticsearch import Elasticsearch, ElasticsearchException except ImportError: - self.log.error("elasticsearch-py not installed.", exc_info=True) + log.error("elasticsearch-py not installed.", exc_info=True) raise unittest.SkipTest("elasticsearch-py not installed.") es = Elasticsearch(settings.HAYSTACK_CONNECTIONS['elasticsearch']['URL']) try: es.info() except ElasticsearchException as e: - self.log.error("elasticsearch not running on %r" % \ + log.error("elasticsearch not running on %r" % \ settings.HAYSTACK_CONNECTIONS['elasticsearch']['URL'], exc_info=True) raise unittest.SkipTest("elasticsearch not running on %r" % \ settings.HAYSTACK_CONNECTIONS['elasticsearch']['URL'], e) From 73be54792df4ba9777728f234c0a86cc7fe6303d Mon Sep 17 00:00:00 2001 From: joaojunior Date: Sun, 24 Jul 2016 19:16:49 +0000 Subject: [PATCH 11/20] Install elasticsearch2.0 via apt --- .travis.yml | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1c007aab1..ea0358f92 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,19 +26,17 @@ addons: before_install: - mkdir -p $HOME/download-cache + - wget -qO - https://packages.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add - - > if [[ $VERSION_ES == '>=2.0.0,<3.0.0' ]]; then - wget https://download.elasticsearch.org/elasticsearch/release/org/elasticsearch/distribution/tar/elasticsearch/2.2.1/elasticsearch-2.2.1.tar.gz - tar zxf elasticsearch-2.2.1.tar.gz - elasticsearch-2.2.1/bin/elasticsearch -d -Dhttp.port=9200 + echo "deb http://packages.elastic.co/elasticsearch/2.0/debian stable main" | sudo tee -a /etc/apt/sources.list.d/elasticsearch-2.0.list else - wget -qO - https://packages.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add - echo "deb http://packages.elastic.co/elasticsearch/1.7/debian stable main" | sudo tee -a /etc/apt/sources.list.d/elasticsearch-1.7.list - sudo apt-get update - sudo apt-get -y install elasticsearch - sudo service elasticsearch restart fi + - sudo apt-get update + - sudo apt-get -y install elasticsearch + - sudo service elasticsearch restart install: - pip install --upgrade setuptools From 3472b06b3690244f043136f8473dd826cdb9bd74 Mon Sep 17 00:00:00 2001 From: joaojunior Date: Sun, 24 Jul 2016 19:44:49 +0000 Subject: [PATCH 12/20] Install elasticsearch2.0 via apt --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index ea0358f92..b37da8625 100644 --- a/.travis.yml +++ b/.travis.yml @@ -30,7 +30,7 @@ before_install: - > if [[ $VERSION_ES == '>=2.0.0,<3.0.0' ]]; then - echo "deb http://packages.elastic.co/elasticsearch/2.0/debian stable main" | sudo tee -a /etc/apt/sources.list.d/elasticsearch-2.0.list + echo "deb http://packages.elastic.co/elasticsearch/2.x/debian stable main" | sudo tee -a /etc/apt/sources.list.d/elasticsearch-2.x.list else echo "deb http://packages.elastic.co/elasticsearch/1.7/debian stable main" | sudo tee -a /etc/apt/sources.list.d/elasticsearch-1.7.list fi From 3ff6c466a6811610ded119ec2117cf2fa3fa64ea Mon Sep 17 00:00:00 2001 From: Bruno Marques Date: Mon, 5 Dec 2016 09:29:03 -0200 Subject: [PATCH 13/20] Fixed expected query behaviour on ES2.x test --- AUTHORS | 1 + test_haystack/elasticsearch2_tests/__init__.py | 2 +- test_haystack/elasticsearch2_tests/test_backend.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/AUTHORS b/AUTHORS index 7fdf08df0..46c970c20 100644 --- a/AUTHORS +++ b/AUTHORS @@ -115,3 +115,4 @@ Thanks to * Tim Babych (@tymofij) for enabling backend-specific parameters in ``.highlight()`` * Antony Raj (@antonyr) for adding endswith input type and fixing contains input type * Morgan Aubert (@ellmetha) for Django 1.10 support + * João Junior (@joaojunior) and Bruno Marques (@ElSaico) for Elasticsearch 2.x support diff --git a/test_haystack/elasticsearch2_tests/__init__.py b/test_haystack/elasticsearch2_tests/__init__.py index b8f51fed2..36d74483f 100644 --- a/test_haystack/elasticsearch2_tests/__init__.py +++ b/test_haystack/elasticsearch2_tests/__init__.py @@ -3,7 +3,7 @@ from django.conf import settings -from ..utils import unittest +import unittest from haystack.utils import log as logging warnings.simplefilter('ignore', Warning) diff --git a/test_haystack/elasticsearch2_tests/test_backend.py b/test_haystack/elasticsearch2_tests/test_backend.py index 14fd3b1aa..f7f9ea861 100644 --- a/test_haystack/elasticsearch2_tests/test_backend.py +++ b/test_haystack/elasticsearch2_tests/test_backend.py @@ -849,7 +849,7 @@ def test_auto_query(self): # This will break horrifically if escaping isn't working. sqs = self.sqs.auto_query('"pants:rule"') self.assertTrue(isinstance(sqs, SearchQuerySet)) - self.assertEqual(repr(sqs.query.query_filter), '') + self.assertEqual(repr(sqs.query.query_filter), '') self.assertEqual(sqs.query.build_query(), u'("pants\\:rule")') self.assertEqual(len(sqs), 0) From db429c2438a51d65daee561de3f206f98e8a46c9 Mon Sep 17 00:00:00 2001 From: Bruno Marques Date: Tue, 6 Dec 2016 10:35:33 -0200 Subject: [PATCH 14/20] Fixed More Like This test with deferred query on Elasticsearch 2.x --- test_haystack/elasticsearch2_tests/test_backend.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test_haystack/elasticsearch2_tests/test_backend.py b/test_haystack/elasticsearch2_tests/test_backend.py index f7f9ea861..90e748098 100644 --- a/test_haystack/elasticsearch2_tests/test_backend.py +++ b/test_haystack/elasticsearch2_tests/test_backend.py @@ -1113,12 +1113,12 @@ def test_more_like_this(self): if hasattr(MockModel.objects, 'defer'): # Make sure MLT works with deferred bits. - mi = MockModel.objects.defer('foo').get(pk=1) - self.assertEqual(mi._deferred, True) - deferred = self.sqs.models(MockModel).more_like_this(mi) - self.assertEqual(deferred.count(), 0) - self.assertEqual([result.pk for result in deferred], []) - self.assertEqual(len([result.pk for result in deferred]), 0) + qs = MockModel.objects.defer('foo') + self.assertEqual(qs.query.deferred_loading[1], True) + deferred = self.sqs.models(MockModel).more_like_this(qs.get(pk=1)) + self.assertEqual(deferred.count(), 10) + self.assertEqual({result.pk for result in deferred}, {u'10', u'5', u'21', u'2', u'4', u'6', u'23', u'9', u'14', u'16'}) + self.assertEqual(len([result.pk for result in deferred]), 10) # Ensure that swapping the ``result_class`` works. self.assertTrue( From 9ab6bfa4af70160c5ec5904ae53dc0a03e3d95cf Mon Sep 17 00:00:00 2001 From: Chris Adams Date: Mon, 2 Jan 2017 13:35:18 -0500 Subject: [PATCH 15/20] Update travis script with ES documentation Add a comment for anyone wondering why this isn't a simple `add-apt-repository` call --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index b37da8625..1b0b723e1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,6 +26,7 @@ addons: before_install: - mkdir -p $HOME/download-cache + # See https://www.elastic.co/guide/en/elasticsearch/reference/current/deb.html#deb-repo - wget -qO - https://packages.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add - - > if [[ $VERSION_ES == '>=2.0.0,<3.0.0' ]]; From a1c33fa564cba857000d4a8484eedc9e3d430dfc Mon Sep 17 00:00:00 2001 From: Chris Adams Date: Mon, 2 Jan 2017 13:49:25 -0500 Subject: [PATCH 16/20] Tests: update ES1 client version check message The name of the Python module changed over time and this now matches the ES2 codebase behaviour of having the error message give you the exact package to install including the version. --- test_haystack/elasticsearch_tests/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test_haystack/elasticsearch_tests/__init__.py b/test_haystack/elasticsearch_tests/__init__.py index 67dda6de6..dd302a090 100644 --- a/test_haystack/elasticsearch_tests/__init__.py +++ b/test_haystack/elasticsearch_tests/__init__.py @@ -17,8 +17,8 @@ def setup(): raise ImportError from elasticsearch import Elasticsearch, ElasticsearchException except ImportError: - log.error("elasticsearch-py not installed.", exc_info=True) - raise unittest.SkipTest("elasticsearch-py not installed.") + log.error("'elasticsearch>=1.0.0,<2.0.0' not installed.", exc_info=True) + raise unittest.SkipTest("'elasticsearch>=1.0.0,<2.0.0' not installed.") es = Elasticsearch(settings.HAYSTACK_CONNECTIONS['elasticsearch']['URL']) try: From 5a5d5e2b4d4f4a1e9085960994811af2e2ed31bb Mon Sep 17 00:00:00 2001 From: Chris Adams Date: Mon, 2 Jan 2017 14:16:32 -0500 Subject: [PATCH 17/20] Tests: update ES version detection in settings This allows the tests to work when run locally or otherwise outside of our Travis / Tox scripts by obtaining the version from the installed `elasticsearch` client library. --- test_haystack/settings.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/test_haystack/settings.py b/test_haystack/settings.py index 55e11bbd2..3e1b2a157 100644 --- a/test_haystack/settings.py +++ b/test_haystack/settings.py @@ -94,10 +94,12 @@ }, } -if os.getenv('VERSION_ES') == ">=2.0.0,<3.0.0": - HAYSTACK_CONNECTIONS['elasticsearch'] = { - 'ENGINE': 'haystack.backends.elasticsearch2_backend.Elasticsearch2SearchEngine', - 'URL': '127.0.0.1:9200/', - 'INDEX_NAME': 'test_default', - 'INCLUDE_SPELLING': True, - } +try: + import elasticsearch + + if (2, ) <= elasticsearch.__version__ <= (3, ): + HAYSTACK_CONNECTIONS['elasticsearch'].update({ + 'ENGINE': 'haystack.backends.elasticsearch2_backend.Elasticsearch2SearchEngine' + }) +except ImportError: + pass From 3c247b6209bd4b79c478ae8db8d4804b58878f36 Mon Sep 17 00:00:00 2001 From: Chris Adams Date: Mon, 2 Jan 2017 14:24:48 -0500 Subject: [PATCH 18/20] Tests: friendlier log message for ES version checks This avoids a potentially scary-looking ImportError flying by in the test output for what's expected in normal usage. --- test_haystack/elasticsearch2_tests/__init__.py | 2 +- test_haystack/elasticsearch_tests/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test_haystack/elasticsearch2_tests/__init__.py b/test_haystack/elasticsearch2_tests/__init__.py index 36d74483f..30b3e8496 100644 --- a/test_haystack/elasticsearch2_tests/__init__.py +++ b/test_haystack/elasticsearch2_tests/__init__.py @@ -17,7 +17,7 @@ def setup(): raise ImportError from elasticsearch import Elasticsearch, exceptions except ImportError: - log.error("'elasticsearch>=2.0.0,<3.0.0' not installed.", exc_info=True) + log.error("Skipping ElasticSearch 2 tests: 'elasticsearch>=2.0.0,<3.0.0' not installed.") raise unittest.SkipTest("'elasticsearch>=2.0.0,<3.0.0' not installed.") url = settings.HAYSTACK_CONNECTIONS['elasticsearch']['URL'] diff --git a/test_haystack/elasticsearch_tests/__init__.py b/test_haystack/elasticsearch_tests/__init__.py index dd302a090..bd33f7b43 100644 --- a/test_haystack/elasticsearch_tests/__init__.py +++ b/test_haystack/elasticsearch_tests/__init__.py @@ -17,7 +17,7 @@ def setup(): raise ImportError from elasticsearch import Elasticsearch, ElasticsearchException except ImportError: - log.error("'elasticsearch>=1.0.0,<2.0.0' not installed.", exc_info=True) + log.error("Skipping ElasticSearch 1 tests: 'elasticsearch>=1.0.0,<2.0.0' not installed.") raise unittest.SkipTest("'elasticsearch>=1.0.0,<2.0.0' not installed.") es = Elasticsearch(settings.HAYSTACK_CONNECTIONS['elasticsearch']['URL']) From 65976474f3b06e0e936830fa02a62325e81bb309 Mon Sep 17 00:00:00 2001 From: Chris Adams Date: Mon, 2 Jan 2017 14:30:24 -0500 Subject: [PATCH 19/20] Tests: avoid unrelated failures when elasticsearch is not installed This avoids spurious failures in tests for other search engines when the elasticsearch client library is not installed at all but the ES backend is still declared in the settings. --- test_haystack/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_haystack/settings.py b/test_haystack/settings.py index 3e1b2a157..f452673c5 100644 --- a/test_haystack/settings.py +++ b/test_haystack/settings.py @@ -102,4 +102,4 @@ 'ENGINE': 'haystack.backends.elasticsearch2_backend.Elasticsearch2SearchEngine' }) except ImportError: - pass + del HAYSTACK_CONNECTIONS['elasticsearch'] From 8712d8903481756a2c008ed65c5dca8cf162d2af Mon Sep 17 00:00:00 2001 From: Chris Adams Date: Mon, 2 Jan 2017 14:39:10 -0500 Subject: [PATCH 20/20] Docs: update Elasticsearch support status --- README.rst | 2 +- docs/backend_support.rst | 2 +- docs/installing_search_engines.rst | 6 +++--- docs/searchqueryset_api.rst | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/README.rst b/README.rst index d5959e291..b8c47db66 100644 --- a/README.rst +++ b/README.rst @@ -10,7 +10,7 @@ API that allows you to plug in different search backends (such as Solr_, Elasticsearch_, Whoosh_, Xapian_, etc.) without having to modify your code. .. _Solr: http://lucene.apache.org/solr/ -.. _Elasticsearch: http://elasticsearch.org/ +.. _Elasticsearch: https://www.elastic.co/products/elasticsearch .. _Whoosh: https://bitbucket.org/mchaput/whoosh/ .. _Xapian: http://xapian.org/ diff --git a/docs/backend_support.rst b/docs/backend_support.rst index d4be1e660..66695801a 100644 --- a/docs/backend_support.rst +++ b/docs/backend_support.rst @@ -50,7 +50,7 @@ Elasticsearch * Stored (non-indexed) fields * Highlighting * Spatial search -* Requires: elasticsearch-py > 1.0 & Elasticsearch 1.0+ (Neither Elasticsearch 2.X `#1247 `_ nor Elasticsearch 5.X `#1383 `_ are supported yet.) +* Requires: `elasticsearch `_ 1.x or 2.x. Elasticsearch 5.X is currently unsupported: see `#1383 `_. Whoosh ------ diff --git a/docs/installing_search_engines.rst b/docs/installing_search_engines.rst index 7fb42e557..c3f316c64 100644 --- a/docs/installing_search_engines.rst +++ b/docs/installing_search_engines.rst @@ -114,9 +114,9 @@ Official Download Location: http://www.elasticsearch.org/download/ Elasticsearch is Java but comes in a pre-packaged form that requires very little other than the JRE. It's also very performant, scales easily and has -an advanced featureset. Haystack currently only supports ElasticSearch 1.x. -ElasticSearch 2.x is not supported yet, if you would like to help, please see -`#1247 `_. +an advanced featureset. Haystack currently only supports Elasticsearch 1.x and 2.x. +Elasticsearch 5.x is not supported yet, if you would like to help, please see +`#1383 `_. Installation is best done using a package manager:: diff --git a/docs/searchqueryset_api.rst b/docs/searchqueryset_api.rst index d4f29d294..ea8e5bbde 100644 --- a/docs/searchqueryset_api.rst +++ b/docs/searchqueryset_api.rst @@ -304,7 +304,7 @@ Example:: # For SOLR (setting f.author.facet.*; see http://wiki.apache.org/solr/SimpleFacetParameters#Parameters) SearchQuerySet().facet('author', mincount=1, limit=10) - # For ElasticSearch (see http://www.elasticsearch.org/guide/reference/api/search/facets/terms-facet.html) + # For Elasticsearch (see http://www.elasticsearch.org/guide/reference/api/search/facets/terms-facet.html) SearchQuerySet().facet('author', size=10, order='term') In the search results you get back, facet counts will be populated in the