make some DB queries more efficient

The queries to build the home page timeline, and to get the lists of
extensions and custom part types in the question editor, were doing
outer joins.

The problem is this pattern:

    Model.objects.filter(a) | related_model.models.all()

This produces an outer join, which is slow.

Instead, these queries could be written as:

    Model.objects.filter(Q(a) | pk__in=related_model.models.all())

This does an inner join, which is fast.

Author: christianp

accounts/models.py

@@ -78,23 +78,22 @@ def projects(self):
 
     def all_timeline(self):
         nonsticky_broadcasts = SiteBroadcast.objects.visible_now().exclude(sticky=True)
-        nonsticky_broadcast_timelineitems = TimelineItem.objects.filter(object_content_type=ContentType.objects.get_for_model(SiteBroadcast), object_id__in=nonsticky_broadcasts)
+        nonsticky_broadcast_timelineitems_filter = Q(object_content_type=ContentType.objects.get_for_model(SiteBroadcast), object_id__in=nonsticky_broadcasts)
 
         projects = Project.objects.filter(Q(owner=self.user) | Q(pk__in=self.user.individual_accesses.for_model(Project).values('object_id')) | Q(watching_non_members=self.user)).values('pk')
         editoritems = EditorItem.objects.filter(Q(author=self.user) | Q(pk__in=self.user.individual_accesses.for_model(EditorItem).values('object_id'))).values('pk')
         queues = ItemQueue.objects.filter(Q(owner=self.user) | Q(pk__in=self.user.individual_accesses.for_model(ItemQueue).values('object_id'))).values('pk')
 
         items = TimelineItem.objects.filter(
+            nonsticky_broadcast_timelineitems_filter |
             Q(editoritems__in=editoritems) |
             Q(editoritems__project__in=projects) |
             Q(projects__in=projects) |
             Q(item_queue_entries__queue__project__in = projects) |
             Q(item_queue_entries__queue__in = queues) |
             Q(item_queue_entry__queue__project__in = projects) |
             Q(item_queue_entry__queue__in = queues)
-        )
-
-        items = (items | nonsticky_broadcast_timelineitems).order_by('-date')
+        ).order_by('-date')
 
         return items
 

editor/views/question.py

@@ -142,7 +142,7 @@ def pre_save(self, form):
     def get_context_data(self, **kwargs):
         context = super(UpdateView, self).get_context_data(**kwargs)
 
-        extensions = Extension.objects.filter(Extension.filter_can_be_viewed_by(self.request.user)) | self.object.extensions.all()
+        extensions = Extension.objects.filter(Extension.filter_can_be_viewed_by(self.user) | Q(pk__in=self.object.extensions.all()))
         extensions = extensions.distinct().order_by(Lower('name'))
         self.item_json['numbasExtensions'] = context['extensions'] = [e.as_json() for e in extensions]
 
@@ -155,16 +155,15 @@ def get_context_data(self, **kwargs):
         self.item_json['editing_history_used'] = self.object.editoritem.comments.exists() or self.object.editoritem.restore_points.exists()
 
         # get publicly available part types first
-        custom_part_types = CustomPartType.objects.filter(public_availability='always')
+        cpt_query = Q(public_availability='always')
         if not self.request.user.is_anonymous:
             # add in the user's own part types
-            users = [self.request.user]+self.object.editoritem.project.members()
-            custom_part_types |= CustomPartType.objects.filter(CustomPartType.filter_can_be_viewed_by(self.request.user))
+            cpt_query |= CustomPartType.filter_can_be_viewed_by(self.request.user)
         # only show part types ready to use
-        custom_part_types = custom_part_types.filter(ready_to_use=True)
+        cpt_query &= Q(ready_to_use=True)
         # also include part types already in use in this question
-        custom_part_types = custom_part_types | self.object.custom_part_types.all()
-        custom_part_types = custom_part_types.distinct()
+        cpt_query |= Q(pk__in=self.object.custom_part_types.all())
+        custom_part_types = CustomPartType.objects.filter(cpt_query)
         self.item_json['custom_part_types'] = context['custom_part_types'] = [c.as_json() for c in custom_part_types]
 
         self.item_json['resources'] = [r.as_json() for r in self.object.resources.all()]