Browse files

Revert "Pass active backend to index queryset calls (closes #534)"

This reverts commit c5e0ce5.
  • Loading branch information...
1 parent 789a8c3 commit 0228996abe3a175815bae8b446542e2624354615 @ntucker committed Apr 4, 2013
View
2 docs/autocomplete.rst
@@ -42,7 +42,7 @@ Example (continuing from the tutorial)::
def get_model(self):
return Note
- def index_queryset(self, using=None):
+ def index_queryset(self):
"""Used when the entire index for model is updated."""
return Note.objects.filter(pub_date__lte=datetime.datetime.now())
View
2 docs/migration_from_1_to_2.rst
@@ -155,7 +155,7 @@ A converted Haystack 2.X index should look like::
def get_model(self):
return Note
- def index_queryset(self, using=None):
+ def index_queryset(self):
"""Used when the entire index for model is updated."""
return self.get_model().objects.filter(pub_date__lte=datetime.datetime.now())
View
36 docs/multiple_index.rst
@@ -163,39 +163,3 @@ via the ``SearchQuerySet.using`` method::
Note that the models a ``SearchQuerySet`` is trying to pull from must all come
from the same index. Haystack is not able to combine search queries against
different indexes.
-
-
-Custom Index Selection
-======================
-
-If a specific backend has been selected, the ``SearchIndex.index_queryset`` and
-``SearchIndex.read_queryset`` will receive the backend name, giving indexes the
-opportunity to customize the returned queryset.
-
-For example, a site which uses separate indexes for recent items and older
-content might define ``index_queryset`` to filter the items based on date::
-
- def index_queryset(self, using=None):
- qs = Note.objects.all()
- archive_limit = datetime.datetime.now() - datetime.timedelta(days=90)
-
- if using == "archive":
- return qs.filter(pub_date__lte=archive_limit)
- else:
- return qs.filter(pub_date__gte=archive_limit)
-
-
-Multi-lingual Content
----------------------
-
-Most search engines require you to set the language at the index level. For
-example, a multi-lingual site using Solr can use `multiple cores <http://wiki.apache.org/solr/CoreAdmin>`_ and corresponding Haystack
-backends using the language name. Under this scenario, queries are simple::
-
- sqs = SearchQuerySet.using(lang).auto_query(…)
-
-During index updates, the Index's ``index_queryset`` method will need to filter
-the items to avoid sending the wrong content to the search engine::
-
- def index_queryset(self, using=None):
- return Post.objects.filter(language=using)
View
8 docs/searchindex_api.rst
@@ -34,7 +34,7 @@ For the impatient::
def get_model(self):
return Note
- def index_queryset(self, using=None):
+ def index_queryset(self):
"Used when the entire index for model is updated."
return self.get_model().objects.filter(pub_date__lte=datetime.datetime.now())
@@ -386,7 +386,7 @@ This method is required & you must override it to return the correct class.
``index_queryset``
------------------
-.. method:: SearchIndex.index_queryset(self, using=None)
+.. method:: SearchIndex.index_queryset(self)
Get the default QuerySet to index when doing a full update.
@@ -395,7 +395,7 @@ Subclasses can override this method to avoid indexing certain objects.
``read_queryset``
-----------------
-.. method:: SearchIndex.read_queryset(self, using=None)
+.. method:: SearchIndex.read_queryset(self)
Get the default QuerySet for read actions.
@@ -609,7 +609,7 @@ For the impatient::
fields = ['user', 'pub_date']
# Note that regular ``SearchIndex`` methods apply.
- def index_queryset(self, using=None):
+ def index_queryset(self):
"Used when the entire index for model is updated."
return Note.objects.filter(pub_date__lte=datetime.datetime.now())
View
2 docs/tutorial.rst
@@ -221,7 +221,7 @@ Haystack to automatically pick it up. The ``NoteIndex`` should look like::
def get_model(self):
return Note
- def index_queryset(self, using=None):
+ def index_queryset(self):
"""Used when the entire index for model is updated."""
return self.get_model().objects.filter(pub_date__lte=datetime.datetime.now())
View
2 example_project/regular_app/search_indexes.py
@@ -19,7 +19,7 @@ class DogIndex(indexes.SearchIndex, indexes.Indexable):
def get_model(self):
return Dog
- def index_queryset(self, using=None):
+ def index_queryset(self):
return self.get_model().objects.filter(public=True)
def prepare_toys(self, obj):
View
12 haystack/indexes.py
@@ -76,7 +76,7 @@ class NoteIndex(indexes.SearchIndex, indexes.Indexable):
def get_model(self):
return Note
- def index_queryset(self, using=None):
+ def index_queryset(self):
return self.get_model().objects.filter(pub_date__lte=datetime.datetime.now())
"""
@@ -102,24 +102,24 @@ def get_model(self):
"""
raise NotImplementedError("You must provide a 'model' method for the '%r' index." % self)
- def index_queryset(self, using=None):
+ def index_queryset(self):
"""
Get the default QuerySet to index when doing a full update.
Subclasses can override this method to avoid indexing certain objects.
"""
return self.get_model()._default_manager.all()
- def read_queryset(self, using=None):
+ def read_queryset(self):
"""
Get the default QuerySet for read actions.
Subclasses can override this method to work with other managers.
Useful when working with default managers that filter some objects.
"""
- return self.index_queryset(using=using)
+ return self.index_queryset()
- def build_queryset(self, using=None, start_date=None, end_date=None):
+ def build_queryset(self, start_date=None, end_date=None):
"""
Get the default QuerySet to index when doing an index update.
@@ -154,7 +154,7 @@ def build_queryset(self, using=None, start_date=None, end_date=None):
warnings.warn("'SearchIndex.get_queryset' was deprecated in Haystack v2. Please rename the method 'index_queryset'.")
index_qs = self.get_queryset()
else:
- index_qs = self.index_queryset(using=using)
+ index_qs = self.index_queryset()
if not hasattr(index_qs, 'filter'):
raise ImproperlyConfigured("The '%r' class must return a 'QuerySet' in the 'index_queryset' method." % self)
View
34 haystack/management/commands/clear_index.py
@@ -1,7 +1,7 @@
from optparse import make_option
import sys
-
from django.core.management.base import BaseCommand
+from haystack.constants import DEFAULT_ALIAS
class Command(BaseCommand):
@@ -10,41 +10,35 @@ class Command(BaseCommand):
make_option('--noinput', action='store_false', dest='interactive', default=True,
help='If provided, no prompts will be issued to the user and the data will be wiped out.'
),
- make_option("-u", "--using", action="append", dest="using",
- default=[],
- help='Update only the named backend (can be used multiple times). '
- 'By default all backends will be updated.'
+ make_option("-u", "--using", action="store", type="string", dest="using", default=DEFAULT_ALIAS,
+ help='If provided, chooses a connection to work with.'
),
)
option_list = BaseCommand.option_list + base_options
-
+
def handle(self, **options):
"""Clears out the search index completely."""
from haystack import connections
self.verbosity = int(options.get('verbosity', 1))
-
- using = options.get('using')
- if not using:
- using = connections.connections_info.keys()
-
+ self.using = options.get('using')
+
if options.get('interactive', True):
print
- print "WARNING: This will irreparably remove EVERYTHING from your search index in connection '%s'." % "', '".join(using)
+ print "WARNING: This will irreparably remove EVERYTHING from your search index in connection '%s'." % self.using
print "Your choices after this are to restore from backups or rebuild via the `rebuild_index` command."
-
+
yes_or_no = raw_input("Are you sure you wish to continue? [y/N] ")
print
-
+
if not yes_or_no.lower().startswith('y'):
print "No action taken."
sys.exit()
-
+
if self.verbosity >= 1:
print "Removing all documents from your index because you said so."
-
- for backend_name in using:
- backend = connections[backend_name].get_backend()
- backend.clear()
-
+
+ backend = connections[self.using].get_backend()
+ backend.clear()
+
if self.verbosity >= 1:
print "All documents removed."
View
48 haystack/management/commands/update_index.py
@@ -1,7 +1,7 @@
from datetime import timedelta
from optparse import make_option
-import logging
import os
+import warnings
from django import db
from django.conf import settings
@@ -11,6 +11,7 @@
from django.utils.encoding import smart_str, force_unicode
from haystack import connections as haystack_connections
+from haystack.constants import DEFAULT_ALIAS
from haystack.query import SearchQuerySet
try:
@@ -70,9 +71,9 @@ def do_update(backend, index, qs, start, end, total, verbosity=1):
if verbosity >= 2:
if hasattr(os, 'getppid') and os.getpid() == os.getppid():
- print " indexed %s - %d of %d." % (start + 1, end, total)
+ print " indexed %s - %d of %d." % (start+1, end, total)
else:
- print " indexed %s - %d of %d (by %s)." % (start + 1, end, total, os.getpid())
+ print " indexed %s - %d of %d (by %s)." % (start+1, end, total, os.getpid())
# FIXME: Get the right backend.
backend.update(index, current_qs)
@@ -120,10 +121,8 @@ class Command(LabelCommand):
make_option('-r', '--remove', action='store_true', dest='remove',
default=False, help='Remove objects from the index that are no longer present in the database.'
),
- make_option("-u", "--using", action="append", dest="using",
- default=[],
- help='Update only the named backend (can be used multiple times). '
- 'By default all backends will be updated.'
+ make_option("-u", "--using", action="store", type="string", dest="using", default=DEFAULT_ALIAS,
+ help='If provided, chooses a connection to work with.'
),
make_option('-k', '--workers', action='store', dest='workers',
default=0, type='int',
@@ -138,11 +137,9 @@ def handle(self, *items, **options):
self.start_date = None
self.end_date = None
self.remove = options.get('remove', False)
+ self.using = options.get('using')
self.workers = int(options.get('workers', 0))
-
- self.backends = options.get('using')
- if not self.backends:
- self.backends = haystack_connections.connections_info.keys()
+ self.backend = haystack_connections[self.using].get_backend()
age = options.get('age', DEFAULT_AGE)
start_date = options.get('start_date')
@@ -205,18 +202,9 @@ def get_models(self, label):
return [get_model(app_label, model_name)]
def handle_label(self, label, **options):
- for using in self.backends:
- try:
- self.update_backend(label, using)
- except:
- logging.exception("Error updating %s using %s ", label, using)
- raise
-
- def update_backend(self, label, using):
from haystack.exceptions import NotHandled
- backend = haystack_connections[using].get_backend()
- unified_index = haystack_connections[using].get_unified_index()
+ unified_index = haystack_connections[self.using].get_unified_index()
if self.workers > 0:
import multiprocessing
@@ -230,21 +218,17 @@ def update_backend(self, label, using):
continue
if self.workers > 0:
- # workers resetting connections leads to references to models / connections getting
- # stale and having their connection disconnected from under them. Resetting before
- # the loop continues and it accesses the ORM makes it better.
+ # workers resetting connections leads to references to models / connections getting stale and having their connection disconnected from under them. Resetting before the loop continues and it accesses the ORM makes it better.
db.close_connection()
- qs = index.build_queryset(using=using, start_date=self.start_date,
- end_date=self.end_date)
-
+ qs = index.build_queryset(start_date=self.start_date, end_date=self.end_date)
total = qs.count()
if self.verbosity >= 1:
print "Indexing %d %s." % (total, force_unicode(model._meta.verbose_name_plural))
pks_seen = set([smart_str(pk) for pk in qs.values_list('pk', flat=True)])
- batch_size = self.batchsize or backend.batch_size
+ batch_size = self.batchsize or self.backend.batch_size
if self.workers > 0:
ghetto_queue = []
@@ -253,9 +237,9 @@ def update_backend(self, label, using):
end = min(start + batch_size, total)
if self.workers == 0:
- do_update(backend, index, qs, start, end, total, self.verbosity)
+ do_update(self.backend, index, qs, start, end, total, self.verbosity)
else:
- ghetto_queue.append(('do_update', model, start, end, total, using, self.start_date, self.end_date, self.verbosity))
+ ghetto_queue.append(('do_update', model, start, end, total, self.using, self.start_date, self.end_date, self.verbosity))
if self.workers > 0:
pool = multiprocessing.Pool(self.workers)
@@ -277,9 +261,9 @@ def update_backend(self, label, using):
upper_bound = start + batch_size
if self.workers == 0:
- do_remove(backend, index, model, pks_seen, start, upper_bound)
+ do_remove(self.backend, index, model, pks_seen, start, upper_bound)
else:
- ghetto_queue.append(('do_remove', model, pks_seen, start, upper_bound, using, self.verbosity))
+ ghetto_queue.append(('do_remove', model, pks_seen, start, upper_bound, self.using, self.verbosity))
if self.workers > 0:
pool = multiprocessing.Pool(self.workers)
View
2 haystack/query.py
@@ -205,7 +205,7 @@ def post_process_results(self, results):
try:
ui = connections[self.query._using].get_unified_index()
index = ui.get_index(model)
- objects = index.read_queryset(using=self.query._using)
+ objects = index.read_queryset()
loaded_objects[model] = objects.in_bulk(models_pks[model])
except NotHandled:
self.log.warning("Model '%s.%s' not handled by the routers.", self.app_label, self.model_name)
View
1 tests/core/tests/__init__.py
@@ -14,4 +14,3 @@
from core.tests.templatetags import *
from core.tests.views import *
from core.tests.utils import *
-from core.tests.management_commands import *
View
12 tests/core/tests/indexes.py
@@ -67,10 +67,10 @@ def load_all_queryset(self):
def get_model(self):
return MockModel
- def index_queryset(self, using=None):
+ def index_queryset(self):
return MockModel.objects.all()
- def read_queryset(self, using=None):
+ def read_queryset(self):
return MockModel.objects.filter(author__in=['daniel1', 'daniel3'])
def build_queryset(self, start_date=None, end_date=None):
@@ -525,11 +525,11 @@ class GhettoAFifthMockModelSearchIndex(indexes.SearchIndex, indexes.Indexable):
def get_model(self):
return AFifthMockModel
- def index_queryset(self, using=None):
+ def index_queryset(self):
# Index everything,
return self.get_model().objects.complete_set()
- def read_queryset(self, using=None):
+ def read_queryset(self):
return self.get_model().objects.all()
@@ -539,7 +539,7 @@ class ReadQuerySetTestSearchIndex(indexes.SearchIndex, indexes.Indexable):
def get_model(self):
return AFifthMockModel
- def read_queryset(self, using=None):
+ def read_queryset(self):
return self.get_model().objects.complete_set()
@@ -549,7 +549,7 @@ class TextReadQuerySetTestSearchIndex(indexes.SearchIndex, indexes.Indexable):
def get_model(self):
return AFifthMockModel
- def read_queryset(self, using=None):
+ def read_queryset(self):
return self.get_model().objects.complete_set()
View
57 tests/core/tests/management_commands.py
@@ -1,57 +0,0 @@
-from mock import patch, call
-
-from django.core.management import call_command
-from django.test import TestCase
-
-__all__ = ['CoreManagementCommandsTestCase']
-
-
-class CoreManagementCommandsTestCase(TestCase):
- @patch("haystack.management.commands.update_index.Command.update_backend")
- def test_update_index_default_using(self, m):
- """update_index uses default index when --using is not present"""
- call_command('update_index')
- m.assert_called_with("core", 'default')
-
- @patch("haystack.management.commands.update_index.Command.update_backend")
- def test_update_index_using(self, m):
- """update_index only applies to indexes specified with --using"""
- call_command('update_index', verbosity=0, using=["eng", "fra"])
- m.assert_any_call("core", "eng")
- m.assert_any_call("core", "fra")
- self.assertTrue(call("core", "default") not in m.call_args_list,
- "update_index should have been restricted to the index specified with --using")
-
- @patch("haystack.loading.ConnectionHandler.__getitem__")
- def test_clear_index_default_using(self, m):
- """clear_index uses default index when --using is not present"""
- call_command('clear_index', verbosity=0, interactive=False)
- m.assert_called_with("default")
-
- @patch("haystack.loading.ConnectionHandler.__getitem__")
- def test_clear_index_using(self, m):
- """clear_index only applies to indexes specified with --using"""
-
- call_command('clear_index', verbosity=0, interactive=False, using=["eng"])
- m.assert_called_with("eng")
- self.assertTrue(m.return_value.get_backend.called, "backend.clear() should be called")
- self.assertTrue(call("default") not in m.call_args_list,
- "clear_index should have been restricted to the index specified with --using")
-
- @patch("haystack.loading.ConnectionHandler.__getitem__")
- @patch("haystack.management.commands.update_index.Command.update_backend")
- def test_rebuild_index_default_using(self, m1, m2):
- """rebuild_index uses default index when --using is not present"""
-
- call_command('rebuild_index', verbosity=0, interactive=False)
- m2.assert_called_with("default")
- m1.assert_any_call("core", "default")
-
- @patch("haystack.loading.ConnectionHandler.__getitem__")
- @patch("haystack.management.commands.update_index.Command.update_backend")
- def test_rebuild_index_using(self, m1, m2):
- """rebuild_index passes --using to clear_index and update_index"""
-
- call_command('rebuild_index', verbosity=0, interactive=False, using=["eng"])
- m2.assert_called_with("eng")
- m1.assert_any_call("core", "eng")
View
4 tests/multipleindex/models.py
@@ -4,14 +4,14 @@
class Foo(models.Model):
title = models.CharField(max_length=255)
body = models.TextField()
-
+
def __unicode__(self):
return self.title
class Bar(models.Model):
author = models.CharField(max_length=255)
content = models.TextField()
-
+
def __unicode__(self):
return self.author
View
6 tests/multipleindex/search_indexes.py
@@ -11,11 +11,7 @@ def get_model(self):
class FooIndex(BaseIndex, indexes.Indexable):
- def index_queryset(self, using=None):
- qs = super(FooIndex, self).index_queryset(using=using)
- if using == "filtered_whoosh":
- qs = qs.filter(body__contains="1")
- return qs
+ pass
# Import the old way & make sure things don't explode.
View
15 tests/multipleindex/tests.py
@@ -1,16 +1,13 @@
import os
import shutil
-
from django.conf import settings
from django.db import models
from django.test import TestCase
-
from haystack import connections, connection_router
from haystack.exceptions import NotHandled
from haystack.query import SearchQuerySet
from haystack.signals import BaseSignalProcessor, RealtimeSignalProcessor
from haystack.utils.loading import UnifiedIndex
-
from multipleindex.search_indexes import FooIndex
from multipleindex.models import Foo, Bar
@@ -32,7 +29,6 @@ def setUp(self):
self.bi = self.ui.get_index(Bar)
self.solr_backend = connections['default'].get_backend()
self.whoosh_backend = connections['whoosh'].get_backend()
- self.filtered_whoosh_backend = connections['filtered_whoosh'].get_backend()
foo_1 = Foo.objects.create(
title='Haystack test',
@@ -188,17 +184,6 @@ def test_excluded_indexes(self):
# Should error, since it's not present.
self.assertRaises(NotHandled, wui.get_index, Bar)
- def test_filtered_index_update(self):
- for i in ('whoosh', 'filtered_whoosh'):
- self.fi.clear(using=i)
- self.fi.update(using=i)
-
- results = self.whoosh_backend.search('foo')
- self.assertEqual(results['hits'], 2)
-
- results = self.filtered_whoosh_backend.search('foo')
- self.assertEqual(results['hits'], 1, "Filtered backend should only contain one record")
-
class TestSignalProcessor(BaseSignalProcessor):
def setup(self):
View
5 tests/multipleindex_settings.py
@@ -15,11 +15,6 @@
'PATH': mkdtemp(prefix='haystack-multipleindex-whoosh-tests-'),
'EXCLUDED_INDEXES': ['multipleindex.search_indexes.BarIndex'],
},
- 'filtered_whoosh': {
- 'ENGINE': 'haystack.backends.whoosh_backend.WhooshEngine',
- 'PATH': mkdtemp(prefix='haystack-multipleindex-filtered-whoosh-tests-'),
- 'EXCLUDED_INDEXES': ['multipleindex.search_indexes.BarIndex'],
- },
}
HAYSTACK_SIGNAL_PROCESSOR = 'haystack.signals.RealtimeSignalProcessor'

0 comments on commit 0228996

Please sign in to comment.