From 0228996abe3a175815bae8b446542e2624354615 Mon Sep 17 00:00:00 2001 From: Nathaniel Tucker Date: Thu, 4 Apr 2013 14:00:31 -0700 Subject: [PATCH] Revert "Pass active backend to index queryset calls (closes #534)" This reverts commit c5e0ce5221fc97f6a9a6fd9d6b6fad6aec960842. --- docs/autocomplete.rst | 2 +- docs/migration_from_1_to_2.rst | 2 +- docs/multiple_index.rst | 36 ------------ docs/searchindex_api.rst | 8 +-- docs/tutorial.rst | 2 +- example_project/regular_app/search_indexes.py | 2 +- haystack/indexes.py | 12 ++-- haystack/management/commands/clear_index.py | 34 +++++------ haystack/management/commands/update_index.py | 48 ++++++---------- haystack/query.py | 2 +- tests/core/tests/__init__.py | 1 - tests/core/tests/indexes.py | 12 ++-- tests/core/tests/management_commands.py | 57 ------------------- tests/multipleindex/models.py | 4 +- tests/multipleindex/search_indexes.py | 6 +- tests/multipleindex/tests.py | 15 ----- tests/multipleindex_settings.py | 5 -- 17 files changed, 54 insertions(+), 194 deletions(-) delete mode 100644 tests/core/tests/management_commands.py diff --git a/docs/autocomplete.rst b/docs/autocomplete.rst index 1872407c1..7c3cbc526 100644 --- a/docs/autocomplete.rst +++ b/docs/autocomplete.rst @@ -42,7 +42,7 @@ Example (continuing from the tutorial):: def get_model(self): return Note - def index_queryset(self, using=None): + def index_queryset(self): """Used when the entire index for model is updated.""" return Note.objects.filter(pub_date__lte=datetime.datetime.now()) diff --git a/docs/migration_from_1_to_2.rst b/docs/migration_from_1_to_2.rst index 6159e06fe..1e019915a 100644 --- a/docs/migration_from_1_to_2.rst +++ b/docs/migration_from_1_to_2.rst @@ -155,7 +155,7 @@ A converted Haystack 2.X index should look like:: def get_model(self): return Note - def index_queryset(self, using=None): + def index_queryset(self): """Used when the entire index for model is updated.""" return self.get_model().objects.filter(pub_date__lte=datetime.datetime.now()) diff --git a/docs/multiple_index.rst b/docs/multiple_index.rst index c51b7341b..7bacf3862 100644 --- a/docs/multiple_index.rst +++ b/docs/multiple_index.rst @@ -163,39 +163,3 @@ via the ``SearchQuerySet.using`` method:: Note that the models a ``SearchQuerySet`` is trying to pull from must all come from the same index. Haystack is not able to combine search queries against different indexes. - - -Custom Index Selection -====================== - -If a specific backend has been selected, the ``SearchIndex.index_queryset`` and -``SearchIndex.read_queryset`` will receive the backend name, giving indexes the -opportunity to customize the returned queryset. - -For example, a site which uses separate indexes for recent items and older -content might define ``index_queryset`` to filter the items based on date:: - - def index_queryset(self, using=None): - qs = Note.objects.all() - archive_limit = datetime.datetime.now() - datetime.timedelta(days=90) - - if using == "archive": - return qs.filter(pub_date__lte=archive_limit) - else: - return qs.filter(pub_date__gte=archive_limit) - - -Multi-lingual Content ---------------------- - -Most search engines require you to set the language at the index level. For -example, a multi-lingual site using Solr can use `multiple cores `_ and corresponding Haystack -backends using the language name. Under this scenario, queries are simple:: - - sqs = SearchQuerySet.using(lang).auto_query(…) - -During index updates, the Index's ``index_queryset`` method will need to filter -the items to avoid sending the wrong content to the search engine:: - - def index_queryset(self, using=None): - return Post.objects.filter(language=using) diff --git a/docs/searchindex_api.rst b/docs/searchindex_api.rst index e55c8c5b0..792b10021 100644 --- a/docs/searchindex_api.rst +++ b/docs/searchindex_api.rst @@ -34,7 +34,7 @@ For the impatient:: def get_model(self): return Note - def index_queryset(self, using=None): + def index_queryset(self): "Used when the entire index for model is updated." return self.get_model().objects.filter(pub_date__lte=datetime.datetime.now()) @@ -386,7 +386,7 @@ This method is required & you must override it to return the correct class. ``index_queryset`` ------------------ -.. method:: SearchIndex.index_queryset(self, using=None) +.. method:: SearchIndex.index_queryset(self) Get the default QuerySet to index when doing a full update. @@ -395,7 +395,7 @@ Subclasses can override this method to avoid indexing certain objects. ``read_queryset`` ----------------- -.. method:: SearchIndex.read_queryset(self, using=None) +.. method:: SearchIndex.read_queryset(self) Get the default QuerySet for read actions. @@ -609,7 +609,7 @@ For the impatient:: fields = ['user', 'pub_date'] # Note that regular ``SearchIndex`` methods apply. - def index_queryset(self, using=None): + def index_queryset(self): "Used when the entire index for model is updated." return Note.objects.filter(pub_date__lte=datetime.datetime.now()) diff --git a/docs/tutorial.rst b/docs/tutorial.rst index 52beb7300..65dc0a0fd 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -221,7 +221,7 @@ Haystack to automatically pick it up. The ``NoteIndex`` should look like:: def get_model(self): return Note - def index_queryset(self, using=None): + def index_queryset(self): """Used when the entire index for model is updated.""" return self.get_model().objects.filter(pub_date__lte=datetime.datetime.now()) diff --git a/example_project/regular_app/search_indexes.py b/example_project/regular_app/search_indexes.py index cc8398a24..6746ef9e0 100644 --- a/example_project/regular_app/search_indexes.py +++ b/example_project/regular_app/search_indexes.py @@ -19,7 +19,7 @@ class DogIndex(indexes.SearchIndex, indexes.Indexable): def get_model(self): return Dog - def index_queryset(self, using=None): + def index_queryset(self): return self.get_model().objects.filter(public=True) def prepare_toys(self, obj): diff --git a/haystack/indexes.py b/haystack/indexes.py index 1f7e4e2a5..8c201765f 100644 --- a/haystack/indexes.py +++ b/haystack/indexes.py @@ -76,7 +76,7 @@ class NoteIndex(indexes.SearchIndex, indexes.Indexable): def get_model(self): return Note - def index_queryset(self, using=None): + def index_queryset(self): return self.get_model().objects.filter(pub_date__lte=datetime.datetime.now()) """ @@ -102,7 +102,7 @@ def get_model(self): """ raise NotImplementedError("You must provide a 'model' method for the '%r' index." % self) - def index_queryset(self, using=None): + def index_queryset(self): """ Get the default QuerySet to index when doing a full update. @@ -110,16 +110,16 @@ def index_queryset(self, using=None): """ return self.get_model()._default_manager.all() - def read_queryset(self, using=None): + def read_queryset(self): """ Get the default QuerySet for read actions. Subclasses can override this method to work with other managers. Useful when working with default managers that filter some objects. """ - return self.index_queryset(using=using) + return self.index_queryset() - def build_queryset(self, using=None, start_date=None, end_date=None): + def build_queryset(self, start_date=None, end_date=None): """ Get the default QuerySet to index when doing an index update. @@ -154,7 +154,7 @@ def build_queryset(self, using=None, start_date=None, end_date=None): warnings.warn("'SearchIndex.get_queryset' was deprecated in Haystack v2. Please rename the method 'index_queryset'.") index_qs = self.get_queryset() else: - index_qs = self.index_queryset(using=using) + index_qs = self.index_queryset() if not hasattr(index_qs, 'filter'): raise ImproperlyConfigured("The '%r' class must return a 'QuerySet' in the 'index_queryset' method." % self) diff --git a/haystack/management/commands/clear_index.py b/haystack/management/commands/clear_index.py index 179b61ed8..d99fb5bb3 100644 --- a/haystack/management/commands/clear_index.py +++ b/haystack/management/commands/clear_index.py @@ -1,7 +1,7 @@ from optparse import make_option import sys - from django.core.management.base import BaseCommand +from haystack.constants import DEFAULT_ALIAS class Command(BaseCommand): @@ -10,41 +10,35 @@ class Command(BaseCommand): make_option('--noinput', action='store_false', dest='interactive', default=True, help='If provided, no prompts will be issued to the user and the data will be wiped out.' ), - make_option("-u", "--using", action="append", dest="using", - default=[], - help='Update only the named backend (can be used multiple times). ' - 'By default all backends will be updated.' + make_option("-u", "--using", action="store", type="string", dest="using", default=DEFAULT_ALIAS, + help='If provided, chooses a connection to work with.' ), ) option_list = BaseCommand.option_list + base_options - + def handle(self, **options): """Clears out the search index completely.""" from haystack import connections self.verbosity = int(options.get('verbosity', 1)) - - using = options.get('using') - if not using: - using = connections.connections_info.keys() - + self.using = options.get('using') + if options.get('interactive', True): print - print "WARNING: This will irreparably remove EVERYTHING from your search index in connection '%s'." % "', '".join(using) + print "WARNING: This will irreparably remove EVERYTHING from your search index in connection '%s'." % self.using print "Your choices after this are to restore from backups or rebuild via the `rebuild_index` command." - + yes_or_no = raw_input("Are you sure you wish to continue? [y/N] ") print - + if not yes_or_no.lower().startswith('y'): print "No action taken." sys.exit() - + if self.verbosity >= 1: print "Removing all documents from your index because you said so." - - for backend_name in using: - backend = connections[backend_name].get_backend() - backend.clear() - + + backend = connections[self.using].get_backend() + backend.clear() + if self.verbosity >= 1: print "All documents removed." diff --git a/haystack/management/commands/update_index.py b/haystack/management/commands/update_index.py index 76447beee..0f6620bc1 100755 --- a/haystack/management/commands/update_index.py +++ b/haystack/management/commands/update_index.py @@ -1,7 +1,7 @@ from datetime import timedelta from optparse import make_option -import logging import os +import warnings from django import db from django.conf import settings @@ -11,6 +11,7 @@ from django.utils.encoding import smart_str, force_unicode from haystack import connections as haystack_connections +from haystack.constants import DEFAULT_ALIAS from haystack.query import SearchQuerySet try: @@ -70,9 +71,9 @@ def do_update(backend, index, qs, start, end, total, verbosity=1): if verbosity >= 2: if hasattr(os, 'getppid') and os.getpid() == os.getppid(): - print " indexed %s - %d of %d." % (start + 1, end, total) + print " indexed %s - %d of %d." % (start+1, end, total) else: - print " indexed %s - %d of %d (by %s)." % (start + 1, end, total, os.getpid()) + print " indexed %s - %d of %d (by %s)." % (start+1, end, total, os.getpid()) # FIXME: Get the right backend. backend.update(index, current_qs) @@ -120,10 +121,8 @@ class Command(LabelCommand): make_option('-r', '--remove', action='store_true', dest='remove', default=False, help='Remove objects from the index that are no longer present in the database.' ), - make_option("-u", "--using", action="append", dest="using", - default=[], - help='Update only the named backend (can be used multiple times). ' - 'By default all backends will be updated.' + make_option("-u", "--using", action="store", type="string", dest="using", default=DEFAULT_ALIAS, + help='If provided, chooses a connection to work with.' ), make_option('-k', '--workers', action='store', dest='workers', default=0, type='int', @@ -138,11 +137,9 @@ def handle(self, *items, **options): self.start_date = None self.end_date = None self.remove = options.get('remove', False) + self.using = options.get('using') self.workers = int(options.get('workers', 0)) - - self.backends = options.get('using') - if not self.backends: - self.backends = haystack_connections.connections_info.keys() + self.backend = haystack_connections[self.using].get_backend() age = options.get('age', DEFAULT_AGE) start_date = options.get('start_date') @@ -205,18 +202,9 @@ def get_models(self, label): return [get_model(app_label, model_name)] def handle_label(self, label, **options): - for using in self.backends: - try: - self.update_backend(label, using) - except: - logging.exception("Error updating %s using %s ", label, using) - raise - - def update_backend(self, label, using): from haystack.exceptions import NotHandled - backend = haystack_connections[using].get_backend() - unified_index = haystack_connections[using].get_unified_index() + unified_index = haystack_connections[self.using].get_unified_index() if self.workers > 0: import multiprocessing @@ -230,21 +218,17 @@ def update_backend(self, label, using): continue if self.workers > 0: - # workers resetting connections leads to references to models / connections getting - # stale and having their connection disconnected from under them. Resetting before - # the loop continues and it accesses the ORM makes it better. + # workers resetting connections leads to references to models / connections getting stale and having their connection disconnected from under them. Resetting before the loop continues and it accesses the ORM makes it better. db.close_connection() - qs = index.build_queryset(using=using, start_date=self.start_date, - end_date=self.end_date) - + qs = index.build_queryset(start_date=self.start_date, end_date=self.end_date) total = qs.count() if self.verbosity >= 1: print "Indexing %d %s." % (total, force_unicode(model._meta.verbose_name_plural)) pks_seen = set([smart_str(pk) for pk in qs.values_list('pk', flat=True)]) - batch_size = self.batchsize or backend.batch_size + batch_size = self.batchsize or self.backend.batch_size if self.workers > 0: ghetto_queue = [] @@ -253,9 +237,9 @@ def update_backend(self, label, using): end = min(start + batch_size, total) if self.workers == 0: - do_update(backend, index, qs, start, end, total, self.verbosity) + do_update(self.backend, index, qs, start, end, total, self.verbosity) else: - ghetto_queue.append(('do_update', model, start, end, total, using, self.start_date, self.end_date, self.verbosity)) + ghetto_queue.append(('do_update', model, start, end, total, self.using, self.start_date, self.end_date, self.verbosity)) if self.workers > 0: pool = multiprocessing.Pool(self.workers) @@ -277,9 +261,9 @@ def update_backend(self, label, using): upper_bound = start + batch_size if self.workers == 0: - do_remove(backend, index, model, pks_seen, start, upper_bound) + do_remove(self.backend, index, model, pks_seen, start, upper_bound) else: - ghetto_queue.append(('do_remove', model, pks_seen, start, upper_bound, using, self.verbosity)) + ghetto_queue.append(('do_remove', model, pks_seen, start, upper_bound, self.using, self.verbosity)) if self.workers > 0: pool = multiprocessing.Pool(self.workers) diff --git a/haystack/query.py b/haystack/query.py index 90b45c3d7..38965662f 100644 --- a/haystack/query.py +++ b/haystack/query.py @@ -205,7 +205,7 @@ def post_process_results(self, results): try: ui = connections[self.query._using].get_unified_index() index = ui.get_index(model) - objects = index.read_queryset(using=self.query._using) + objects = index.read_queryset() loaded_objects[model] = objects.in_bulk(models_pks[model]) except NotHandled: self.log.warning("Model '%s.%s' not handled by the routers.", self.app_label, self.model_name) diff --git a/tests/core/tests/__init__.py b/tests/core/tests/__init__.py index 2aef8ad61..da4603983 100644 --- a/tests/core/tests/__init__.py +++ b/tests/core/tests/__init__.py @@ -14,4 +14,3 @@ from core.tests.templatetags import * from core.tests.views import * from core.tests.utils import * -from core.tests.management_commands import * diff --git a/tests/core/tests/indexes.py b/tests/core/tests/indexes.py index f121811c4..29ee880d4 100644 --- a/tests/core/tests/indexes.py +++ b/tests/core/tests/indexes.py @@ -67,10 +67,10 @@ def load_all_queryset(self): def get_model(self): return MockModel - def index_queryset(self, using=None): + def index_queryset(self): return MockModel.objects.all() - def read_queryset(self, using=None): + def read_queryset(self): return MockModel.objects.filter(author__in=['daniel1', 'daniel3']) def build_queryset(self, start_date=None, end_date=None): @@ -525,11 +525,11 @@ class GhettoAFifthMockModelSearchIndex(indexes.SearchIndex, indexes.Indexable): def get_model(self): return AFifthMockModel - def index_queryset(self, using=None): + def index_queryset(self): # Index everything, return self.get_model().objects.complete_set() - def read_queryset(self, using=None): + def read_queryset(self): return self.get_model().objects.all() @@ -539,7 +539,7 @@ class ReadQuerySetTestSearchIndex(indexes.SearchIndex, indexes.Indexable): def get_model(self): return AFifthMockModel - def read_queryset(self, using=None): + def read_queryset(self): return self.get_model().objects.complete_set() @@ -549,7 +549,7 @@ class TextReadQuerySetTestSearchIndex(indexes.SearchIndex, indexes.Indexable): def get_model(self): return AFifthMockModel - def read_queryset(self, using=None): + def read_queryset(self): return self.get_model().objects.complete_set() diff --git a/tests/core/tests/management_commands.py b/tests/core/tests/management_commands.py deleted file mode 100644 index d94919b79..000000000 --- a/tests/core/tests/management_commands.py +++ /dev/null @@ -1,57 +0,0 @@ -from mock import patch, call - -from django.core.management import call_command -from django.test import TestCase - -__all__ = ['CoreManagementCommandsTestCase'] - - -class CoreManagementCommandsTestCase(TestCase): - @patch("haystack.management.commands.update_index.Command.update_backend") - def test_update_index_default_using(self, m): - """update_index uses default index when --using is not present""" - call_command('update_index') - m.assert_called_with("core", 'default') - - @patch("haystack.management.commands.update_index.Command.update_backend") - def test_update_index_using(self, m): - """update_index only applies to indexes specified with --using""" - call_command('update_index', verbosity=0, using=["eng", "fra"]) - m.assert_any_call("core", "eng") - m.assert_any_call("core", "fra") - self.assertTrue(call("core", "default") not in m.call_args_list, - "update_index should have been restricted to the index specified with --using") - - @patch("haystack.loading.ConnectionHandler.__getitem__") - def test_clear_index_default_using(self, m): - """clear_index uses default index when --using is not present""" - call_command('clear_index', verbosity=0, interactive=False) - m.assert_called_with("default") - - @patch("haystack.loading.ConnectionHandler.__getitem__") - def test_clear_index_using(self, m): - """clear_index only applies to indexes specified with --using""" - - call_command('clear_index', verbosity=0, interactive=False, using=["eng"]) - m.assert_called_with("eng") - self.assertTrue(m.return_value.get_backend.called, "backend.clear() should be called") - self.assertTrue(call("default") not in m.call_args_list, - "clear_index should have been restricted to the index specified with --using") - - @patch("haystack.loading.ConnectionHandler.__getitem__") - @patch("haystack.management.commands.update_index.Command.update_backend") - def test_rebuild_index_default_using(self, m1, m2): - """rebuild_index uses default index when --using is not present""" - - call_command('rebuild_index', verbosity=0, interactive=False) - m2.assert_called_with("default") - m1.assert_any_call("core", "default") - - @patch("haystack.loading.ConnectionHandler.__getitem__") - @patch("haystack.management.commands.update_index.Command.update_backend") - def test_rebuild_index_using(self, m1, m2): - """rebuild_index passes --using to clear_index and update_index""" - - call_command('rebuild_index', verbosity=0, interactive=False, using=["eng"]) - m2.assert_called_with("eng") - m1.assert_any_call("core", "eng") diff --git a/tests/multipleindex/models.py b/tests/multipleindex/models.py index 2b8881a38..3205cd5ba 100644 --- a/tests/multipleindex/models.py +++ b/tests/multipleindex/models.py @@ -4,7 +4,7 @@ class Foo(models.Model): title = models.CharField(max_length=255) body = models.TextField() - + def __unicode__(self): return self.title @@ -12,6 +12,6 @@ def __unicode__(self): class Bar(models.Model): author = models.CharField(max_length=255) content = models.TextField() - + def __unicode__(self): return self.author diff --git a/tests/multipleindex/search_indexes.py b/tests/multipleindex/search_indexes.py index 487631238..7eb9c1830 100644 --- a/tests/multipleindex/search_indexes.py +++ b/tests/multipleindex/search_indexes.py @@ -11,11 +11,7 @@ def get_model(self): class FooIndex(BaseIndex, indexes.Indexable): - def index_queryset(self, using=None): - qs = super(FooIndex, self).index_queryset(using=using) - if using == "filtered_whoosh": - qs = qs.filter(body__contains="1") - return qs + pass # Import the old way & make sure things don't explode. diff --git a/tests/multipleindex/tests.py b/tests/multipleindex/tests.py index 1da782586..7e48e3d84 100644 --- a/tests/multipleindex/tests.py +++ b/tests/multipleindex/tests.py @@ -1,16 +1,13 @@ import os import shutil - from django.conf import settings from django.db import models from django.test import TestCase - from haystack import connections, connection_router from haystack.exceptions import NotHandled from haystack.query import SearchQuerySet from haystack.signals import BaseSignalProcessor, RealtimeSignalProcessor from haystack.utils.loading import UnifiedIndex - from multipleindex.search_indexes import FooIndex from multipleindex.models import Foo, Bar @@ -32,7 +29,6 @@ def setUp(self): self.bi = self.ui.get_index(Bar) self.solr_backend = connections['default'].get_backend() self.whoosh_backend = connections['whoosh'].get_backend() - self.filtered_whoosh_backend = connections['filtered_whoosh'].get_backend() foo_1 = Foo.objects.create( title='Haystack test', @@ -188,17 +184,6 @@ def test_excluded_indexes(self): # Should error, since it's not present. self.assertRaises(NotHandled, wui.get_index, Bar) - def test_filtered_index_update(self): - for i in ('whoosh', 'filtered_whoosh'): - self.fi.clear(using=i) - self.fi.update(using=i) - - results = self.whoosh_backend.search('foo') - self.assertEqual(results['hits'], 2) - - results = self.filtered_whoosh_backend.search('foo') - self.assertEqual(results['hits'], 1, "Filtered backend should only contain one record") - class TestSignalProcessor(BaseSignalProcessor): def setup(self): diff --git a/tests/multipleindex_settings.py b/tests/multipleindex_settings.py index 0ae8d23d8..1a2364b6b 100644 --- a/tests/multipleindex_settings.py +++ b/tests/multipleindex_settings.py @@ -15,11 +15,6 @@ 'PATH': mkdtemp(prefix='haystack-multipleindex-whoosh-tests-'), 'EXCLUDED_INDEXES': ['multipleindex.search_indexes.BarIndex'], }, - 'filtered_whoosh': { - 'ENGINE': 'haystack.backends.whoosh_backend.WhooshEngine', - 'PATH': mkdtemp(prefix='haystack-multipleindex-filtered-whoosh-tests-'), - 'EXCLUDED_INDEXES': ['multipleindex.search_indexes.BarIndex'], - }, } HAYSTACK_SIGNAL_PROCESSOR = 'haystack.signals.RealtimeSignalProcessor'