Skip to content

Commit

Permalink
Support index_to argument
Browse files Browse the repository at this point in the history
  • Loading branch information
@dcramer
dcramer committed Sep 1, 2010
1 parent 6e68b65 commit 41f921c
Show file tree
Hide file tree
Showing 2 changed files with 87 additions and 45 deletions.
127 changes: 85 additions & 42 deletions indexer/manager.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,8 @@
class LazyIndexLookup(Proxy):
__slots__ = ('__data__', '__instance__')

def __init__(self, model, model_class, **pairs):
object.__setattr__(self, '__data__', (model, model_class, pairs))
def __init__(self, model, model_class, queryset=None, **pairs):
object.__setattr__(self, '__data__', (model, model_class, queryset, pairs))
object.__setattr__(self, '__instance__', None)

def _get_current_object(self):
Expand All @@ -24,30 +24,40 @@ def _get_current_object(self):
inst = self.__instance__
if inst is not None:
return inst
model, model_class, pairs = self.__data__
model, model_class, qs, pairs = self.__data__

app_label = model_class._meta.app_label
module_name = model_class._meta.module_name
cache_key_base = ':'.join([app_label, module_name])

base_qs = model.objects.filter(app_label=app_label, module_name=module_name)
if qs is None:
qs = model_class.objects.all()

# TODO: this should just do subqueries or something, we can't
# cache 8000000000 keys in one index
list_of_pks = None
tbl = model._meta.db_table
main = model_class._meta.db_table
pk = model_class._meta.pk.column
for column, value in pairs.iteritems():
cache_key = '%s:%s=%s' % (cache_key_base, column, value)
data = cache.get(cache_key)
if data is None:
data = list(base_qs.filter(column=column, value=value).values('object_id'))
data = [d['object_id'] for d in data]
cache.set(cache_key, data)
if list_of_pks is None:
list_of_pks = data
else:
list_of_pks = [d for d in list_of_pks if d in data]

qs = model_class.objects.filter(pk__in=list_of_pks)
#cid = '_i%d' % abs(hash(column))
# print self.model.objects.filter(module_name=module_name, app_label=app_label, column=column, value=value).values_list('object_id')
# qs = qs.filter(pk__in=self.model.objects.filter(module_name=module_name, app_label=app_label, column=column, value=value).values_list('object_id'))
cid = tbl
qs = qs.extra(
# tables=['%s as %s' % (tbl, cid)],
tables=[tbl],
where=['%(cid)s.module_name = %%s and %(cid)s.app_label = %%s and %(cid)s.column = %%s and %(cid)s.value = %%s and %(cid)s.object_id = %(main)s.%(pk)s::varchar' % dict(
cid=cid,
pk=pk,
main=main,
)],
params=[
module_name,
app_label,
column,
value,
],
)
print str(qs.query)

object.__setattr__(self, '__instance__', qs)
return qs
_current_object = property(_get_current_object)
Expand All @@ -57,35 +67,46 @@ def get_for_model(self, model_class, **kwargs):
if len(kwargs) < 1:
raise ValueError

return LazyIndexLookup(self.model, model_class, **kwargs)
return LazyIndexLookup(self.model, model_class, None, **kwargs)

def get_for_queryset(self, queryset, **kwargs):
if len(kwargs) < 1:
raise ValueError

return LazyIndexLookup(self.model, queryset.model, queryset, **kwargs)

def register_model(self, model_class, column):
def register_model(self, model_class, column, index_to=None):
"""Registers a model and an index for it."""
if model_class not in self.model.indexes:
self.model.indexes[model_class] = set([column])
self.model.indexes[model_class] = set([(column, index_to)])
else:
self.model.indexes[model_class].add(column)
self.model.indexes[model_class].add((column, index_to))
signals.post_save.connect(self.model.handle_save, sender=model_class)
signals.pre_delete.connect(self.model.handle_delete, sender=model_class)


def remove_from_index(self, instance):
app_label = instance._meta.app_label
module_name = instance._meta.module_name
tbl = self.model._meta.db_table
self.filter(app_label=app_label, module_name=module_name, object_id=instance.pk).delete()
# TODO: Delete each cache for instance
# cache.delete('%s:%s:%s=%s' % (app_label, module_name, column))

def save_in_index(self, instance, column):
def save_in_index(self, instance, column, index_to=None):
"""Updates an index for an instance.
You may pass column as base__sub to access
values stored deeper in the hierarchy."""
if index_to:
index_to = instance._meta.get_field_by_name(index_to)[0]

if not index_to:
app_label = instance._meta.app_label
module_name = instance._meta.module_name
object_id = instance.pk
else:
app_label = index_to.rel.to._meta.app_label
module_name = index_to.rel.to._meta.module_name
object_id = getattr(instance, index_to.column)

app_label = instance._meta.app_label
module_name = instance._meta.module_name
tbl = self.model._meta.db_table
value = instance
first = True
for bit in column.split(COLUMN_SEPARATOR):
Expand All @@ -95,31 +116,53 @@ def save_in_index(self, instance, column):
else:
value = value.get(bit)
if not value:
self.filter(app_label=app_label, module_name=module_name, object_id=instance.pk, column=column).delete()
self.filter(app_label=app_label, module_name=module_name, object_id=object_id, column=column).delete()
else:
qs = self.filter(app_label=app_label, module_name=module_name, object_id=instance.pk, column=column)
# TODO: in mysql this can be a single operation
qs = self.filter(app_label=app_label, module_name=module_name, object_id=object_id, column=column)
if qs.exists():
qs.update(value=value)
else:
self.create(app_label=app_label, module_name=module_name, object_id=instance.pk, column=column, value=value)
# TODO: this needs to take the original value and wipe its cache as well
cache.delete('%s:%s:%s=%s' % (app_label, module_name, column, value))

def create_index(self, model_class, column):
self.create(app_label=app_label, module_name=module_name, object_id=object_id, column=column, value=value)

def create_index(self, model_class, column, index_to=None):
"""Creates and prepopulates an index.
You may pass column as base__sub to access
values stored deeper in the hierarchy."""

# make sure the index exists
self.register_model(model_class, column)
app_label = model_class._meta.app_label
module_name = model_class._meta.module_name
if index_to:
index_to = model_class._meta.get_field_by_name(index_to)[0]

# must pull from original data
qs = model_class.objects.all()
column_bits = column.split(COLUMN_SEPARATOR)
for m in model_class.objects.all():

if not index_to:
app_label = model_class._meta.app_label
module_name = model_class._meta.module_name
else:
app_label = index_to.rel.to._meta.app_label
module_name = index_to.rel.to._meta.module_name

for m in qs:
if not index_to:
object_id = m.pk
else:
object_id = getattr(m, index_to.column)

value = m
first = True
for bit in column.split(COLUMN_SEPARATOR):
if first:
value = getattr(value, bit)
first = False
else:
value = value.get(bit)
for bit in column_bits:
value = value.get(bit)
if not value:
continue
self.create(app_label=app_label, module_name=module_name, object_id=m.pk, column=column, value=value)
self.create(app_label=app_label, module_name=module_name, object_id=object_id, column=column, value=value)
self.register_model(model_class, column)
5 changes: 2 additions & 3 deletions indexer/models.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,8 @@ def __unicode__(self):
@classmethod
def handle_save(cls, sender, instance, created, **kwargs):
"""Handles updating this model's indexes."""
print sender
for column in Index.indexes[sender]:
cls.objects.save_in_index(instance, column)
for column, index_to in Index.indexes[sender]:
cls.objects.save_in_index(instance, column, index_to)

@classmethod
def handle_delete(cls, sender, instance, **kwargs):
Expand Down

0 comments on commit 41f921c

Please sign in to comment.