docs/utils.rst haystack/utils.py tests/core/tests/utils.py @@ -20,16 +20,6 @@ access this class. Method Reference ================ -``get_identifier`` ------------------- - -.. method:: SearchBackend.get_identifier(self, obj_or_string) - -Get an unique identifier for the object or a string representing the -object. - -If not overridden, uses <app_label>.<object_name>.<pk>. - ``update`` ---------- docs/searchbackend_api.rst @@ -16,7 +16,6 @@ except NameError: from sets import Set as set -IDENTIFIER_REGEX = re.compile('^[\w\d_]+\.[\w\d_]+\.\d+$') VALID_GAPS = ['year', 'month', 'day', 'hour', 'minute', 'second'] @@ -76,21 +75,6 @@ class BaseSearchBackend(object): from haystack import site self.site = site - def get_identifier(self, obj_or_string): - """ - Get an unique identifier for the object or a string representing the - object. - - If not overridden, uses <app_label>.<object_name>.<pk>. - """ - if isinstance(obj_or_string, basestring): - if not IDENTIFIER_REGEX.match(obj_or_string): - raise AttributeError("Provided string '%s' is not a valid identifier." % obj_or_string) - - return obj_or_string - - return u"%s.%s.%s" % (obj_or_string._meta.app_label, obj_or_string._meta.module_name, obj_or_string._get_pk_val()) - def update(self, index, iterable): """ Updates the backend when given a SearchIndex and a collection of haystack/backends/__init__.py @@ -7,6 +7,7 @@ from haystack.backends import BaseSearchBackend, BaseSearchQuery, log_query from haystack.exceptions import MissingDependency, MoreLikeThisError from haystack.fields import DateField, DateTimeField, IntegerField, FloatField, BooleanField, MultiValueField from haystack.models import SearchResult +from haystack.utils import get_identifier try: set except NameError: @@ -50,21 +51,16 @@ class SearchBackend(BaseSearchBackend): try: for obj in iterable: - doc = {} - doc['id'] = self.get_identifier(obj) - doc['django_ct'] = "%s.%s" % (obj._meta.app_label, obj._meta.module_name) - doc['django_id'] = force_unicode(obj.pk) - doc.update(index.prepare(obj)) - docs.append(doc) + docs.append(index.prepare(obj)) except UnicodeDecodeError: sys.stderr.write("Chunk failed.\n") self.conn.add(docs, commit=commit) - + def remove(self, obj_or_string, commit=True): - solr_id = self.get_identifier(obj_or_string) + solr_id = get_identifier(obj_or_string) self.conn.delete(id=solr_id, commit=commit) - + def clear(self, models=[], commit=True): if not models: # *:* matches all docs in Solr @@ -193,7 +189,7 @@ class SearchBackend(BaseSearchBackend): if narrow_queries: params['fq'] = list(narrow_queries) - raw_results = self.conn.more_like_this("id:%s" % self.get_identifier(model_instance), field_name, **params) + raw_results = self.conn.more_like_this("id:%s" % get_identifier(model_instance), field_name, **params) return self._process_results(raw_results) def _process_results(self, raw_results, highlight=False): haystack/backends/solr_backend.py @@ -11,6 +11,7 @@ from haystack.backends import BaseSearchBackend, BaseSearchQuery, log_query from haystack.fields import DateField, DateTimeField, IntegerField, FloatField, BooleanField, MultiValueField from haystack.exceptions import MissingDependency, SearchBackendError from haystack.models import SearchResult +from haystack.utils import get_identifier try: set except NameError: @@ -126,18 +127,13 @@ class SearchBackend(BaseSearchBackend): writer = self.index.writer() for obj in iterable: - doc = {} - doc['id'] = force_unicode(self.get_identifier(obj)) - doc['django_ct'] = force_unicode("%s.%s" % (obj._meta.app_label, obj._meta.module_name)) - doc['django_id'] = force_unicode(obj.pk) - other_data = index.prepare(obj) + doc = index.prepare(obj) # Really make sure it's unicode, because Whoosh won't have it any # other way. - for key in other_data: - other_data[key] = self._from_python(other_data[key]) + for key in doc: + doc[key] = self._from_python(doc[key]) - doc.update(other_data) writer.update_document(**doc) # For now, commit no matter what, as we run into locking issues otherwise. @@ -153,7 +149,7 @@ class SearchBackend(BaseSearchBackend): self.setup() self.index = self.index.refresh() - whoosh_id = self.get_identifier(obj_or_string) + whoosh_id = get_identifier(obj_or_string) self.index.delete_by_query(q=self.parser.parse(u'id:"%s"' % whoosh_id)) # For now, commit no matter what, as we run into locking issues otherwise. haystack/backends/whoosh_backend.py @@ -1,6 +1,8 @@ from django.db.models import signals +from django.utils.encoding import force_unicode import haystack from haystack.fields import * +from haystack.utils import get_identifier class DeclarativeMetaclass(type): @@ -86,7 +88,11 @@ class SearchIndex(object): """ Fetches and adds/alters data before indexing. """ - self.prepared_data = {} + self.prepared_data = { + 'id': get_identifier(obj), + 'django_ct': "%s.%s" % (obj._meta.app_label, obj._meta.module_name), + 'django_id': force_unicode(obj.pk), + } for field_name, field in self.fields.items(): self.prepared_data[field_name] = field.prepare(obj) haystack/indexes.py @@ -1,29 +1,6 @@ -import datetime from django.core.exceptions import ImproperlyConfigured from django.test import TestCase import haystack -from haystack.backends import BaseSearchBackend -from core.models import MockModel, AnotherMockModel - - -class BaseSearchBackendTestCase(TestCase): - def setUp(self): - super(BaseSearchBackendTestCase, self).setUp() - self.bsb = BaseSearchBackend() - - def test_get_identifier(self): - # Various invalid identifiers. - self.assertRaises(AttributeError, self.bsb.get_identifier, 'core') - self.assertRaises(AttributeError, self.bsb.get_identifier, 'core.mockmodel') - self.assertRaises(AttributeError, self.bsb.get_identifier, 'core.mockmodel.foo') - self.assertRaises(AttributeError, self.bsb.get_identifier, 'core-app.mockmodel.1') - - # Valid string identifier. - self.assertEqual(self.bsb.get_identifier('core.mockmodel.1'), 'core.mockmodel.1') - - # Valid object. - mock = MockModel.objects.get(pk=1) - self.assertEqual(self.bsb.get_identifier(mock), 'core.mockmodel.1') class LoadBackendTestCase(TestCase): tests/core/tests/backends.py @@ -122,8 +122,8 @@ class SearchIndexTestCase(TestCase): mock.user = 'daniel%s' % mock.id mock.pub_date = datetime.datetime(2009, 1, 31, 4, 19, 0) - self.assertEqual(len(self.mi.prepare(mock)), 4) - self.assertEqual(sorted(self.mi.prepare(mock).keys()), ['author', 'content', 'extra', 'pub_date']) + self.assertEqual(len(self.mi.prepare(mock)), 7) + self.assertEqual(sorted(self.mi.prepare(mock).keys()), ['author', 'content', 'django_ct', 'django_id', 'extra', 'id', 'pub_date']) def test_custom_prepare(self): mock = MockModel() @@ -131,8 +131,8 @@ class SearchIndexTestCase(TestCase): mock.author = 'daniel%s' % mock.id mock.pub_date = datetime.datetime(2009, 1, 31, 4, 19, 0) - self.assertEqual(len(self.cmi.prepare(mock)), 6) - self.assertEqual(sorted(self.cmi.prepare(mock).keys()), ['author', 'content', 'extra', 'hello', 'pub_date', 'whee']) + self.assertEqual(len(self.cmi.prepare(mock)), 9) + self.assertEqual(sorted(self.cmi.prepare(mock).keys()), ['author', 'content', 'django_ct', 'django_id', 'extra', 'hello', 'id', 'pub_date', 'whee']) def test_custom_prepare_author(self): mock = MockModel() @@ -140,8 +140,8 @@ class SearchIndexTestCase(TestCase): mock.author = 'daniel%s' % mock.id mock.pub_date = datetime.datetime(2009, 1, 31, 4, 19, 0) - self.assertEqual(len(self.cmi.prepare(mock)), 6) - self.assertEqual(sorted(self.cmi.prepare(mock).keys()), ['author', 'content', 'extra', 'hello', 'pub_date', 'whee']) + self.assertEqual(len(self.cmi.prepare(mock)), 9) + self.assertEqual(sorted(self.cmi.prepare(mock).keys()), ['author', 'content', 'django_ct', 'django_id', 'extra', 'hello', 'id', 'pub_date', 'whee']) self.assertEqual(self.cmi.prepared_data['author'], "Hi, I'm daniel20") def test_custom_model_attr(self): @@ -150,8 +150,8 @@ class SearchIndexTestCase(TestCase): mock.author = 'daniel%s' % mock.id mock.pub_date = datetime.datetime(2009, 1, 31, 4, 19, 0) - self.assertEqual(len(self.cmi.prepare(mock)), 6) - self.assertEqual(sorted(self.cmi.prepare(mock).keys()), ['author', 'content', 'extra', 'hello', 'pub_date', 'whee']) + self.assertEqual(len(self.cmi.prepare(mock)), 9) + self.assertEqual(sorted(self.cmi.prepare(mock).keys()), ['author', 'content', 'django_ct', 'django_id', 'extra', 'hello', 'id', 'pub_date', 'whee']) self.assertEqual(self.cmi.prepared_data['hello'], u'World!') def test_get_content_field(self): @@ -233,8 +233,8 @@ class SearchIndexTestCase(TestCase): mock.pub_date = datetime.datetime(2009, 1, 31, 4, 19, 0) prepared_data = self.cnmi.prepare(mock) - self.assertEqual(len(prepared_data), 1) - self.assertEqual(sorted(prepared_data.keys()), ['content']) + self.assertEqual(len(prepared_data), 4) + self.assertEqual(sorted(prepared_data.keys()), ['content', 'django_ct', 'django_id', 'id']) class BasicModelSearchIndex(indexes.ModelSearchIndex): tests/core/tests/indexes.py @@ -2,6 +2,7 @@ from django.db.models.loading import get_model from django.utils.encoding import force_unicode from haystack.backends import BaseSearchBackend, BaseSearchQuery, log_query from haystack.models import SearchResult +from haystack.utils import get_identifier from core.models import MockModel @@ -20,15 +21,11 @@ class MockSearchBackend(BaseSearchBackend): def update(self, index, iterable, commit=True): for obj in iterable: - doc = {} - doc['id'] = self.get_identifier(obj) - doc['django_ct'] = force_unicode("%s.%s" % (obj._meta.app_label, obj._meta.module_name)) - doc['django_id'] = force_unicode(obj.pk) - doc.update(index.prepare(obj)) + doc = index.prepare(obj) self.docs[doc['id']] = doc def remove(self, obj, commit=True): - del(self.docs[self.get_identifier(obj)]) + del(self.docs[get_identifier(obj)]) def clear(self, models=[], commit=True): self.docs = {} tests/core/tests/mocks.py 689ec28a0822519843f1cf88b9dcc8ae7959b310 Daniel Lindsley daniel@toastdriven.com http://github.com/toastdriven/django-haystack/commit/ed8bbe56a7da755e0edbdaf82df18d46f77e3562 ed8bbe56a7da755e0edbdaf82df18d46f77e3562 2009-11-08T00:49:02-08:00 2009-11-08T00:49:02-08:00 Moved Haystack's internal fields out of the backends and into `SearchIndex.prepare`. This is both somewhat more DRY as well as a step toward Haystack being useful to non-Django projects. 6766ee4a12ec36dd0588b1dc91ad842e07d0424b Daniel Lindsley daniel@toastdriven.com