add django-perf-rec module and use it to improve SQL queries in many views/processes

CHANGELOG.rst

@@ -6,6 +6,7 @@ master (unreleased)
 ===================
 
 * Added a make target to install the demo site.
+* Added django-perf-rec module for tests and improved SQL queries in `ContextFormDetailView` (#54, #154, #160).
 
 
 Release 0.5.0 (2017-01-10)

demo/demo/settings.py

@@ -12,6 +12,7 @@
 
 # Build paths inside the project like this: os.path.join(BASE_DIR, ...)
 import os
+import django
 
 BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 
@@ -80,9 +81,14 @@
 # Database
 # https://docs.djangoproject.com/en/1.8/ref/settings/#databases
 
+if django.VERSION[:2] == (1, 8):
+    engine = 'django18_sqlite3_backend'
+else:
+    engine = 'django.db.backends.sqlite3',
+
 DATABASES = {
     'default': {
-        'ENGINE': 'django.db.backends.sqlite3',
+        'ENGINE': engine,
         'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
     }
 }
@@ -122,3 +128,9 @@
 FORMIDABLE_POST_UPDATE_CALLBACK_SUCCESS = 'demo.callback_success_message'
 FORMIDABLE_POST_CREATE_CALLBACK_FAIL = 'demo.callback_fail_message'
 FORMIDABLE_POST_UPDATE_CALLBACK_FAIL = 'demo.callback_fail_message'
+
+
+# django-perf-rec settings
+PERF_REC = {
+    'MODE': 'all'
+}

demo/django18_sqlite3_backend/__init__.py

@@ -0,0 +1,3 @@
+# -*- coding:utf-8 -*-
+# flake8: noqa
+from __future__ import absolute_import, division, print_function, unicode_literals

demo/django18_sqlite3_backend/base.py

@@ -0,0 +1,14 @@
+# -*- coding:utf-8 -*-
+# flake8: noqa
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+from django.db.backends.sqlite3.base import DatabaseWrapper as OrigDatabaseWrapper
+
+from .operations import DatabaseOperations
+
+
+class DatabaseWrapper(OrigDatabaseWrapper):
+
+    def __init__(self, *args, **kwargs):
+        super(DatabaseWrapper, self).__init__(*args, **kwargs)
+        self.ops = DatabaseOperations(self)

demo/django18_sqlite3_backend/operations.py

@@ -0,0 +1,43 @@
+# -*- coding:utf-8 -*-
+# flake8: noqa
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+from django.db.backends.sqlite3.operations import DatabaseOperations as OrigDatabaseOperations
+
+
+class DatabaseOperations(OrigDatabaseOperations):
+
+    # From Django commit 4f6a7663bcddffb114f2647f9928cbf1fdd8e4b5
+
+    def _quote_params_for_last_executed_query(self, params):
+        """
+        Only for last_executed_query! Don't use this to execute SQL queries!
+        """
+        sql = 'SELECT ' + ', '.join(['QUOTE(?)'] * len(params))
+        # Bypass Django's wrappers and use the underlying sqlite3 connection
+        # to avoid logging this query - it would trigger infinite recursion.
+        cursor = self.connection.connection.cursor()
+        # Native sqlite3 cursors cannot be used as context managers.
+        try:
+            return cursor.execute(sql, params).fetchone()
+        finally:
+            cursor.close()
+
+    def last_executed_query(self, cursor, sql, params):
+        # Python substitutes parameters in Modules/_sqlite/cursor.c with:
+        # pysqlite_statement_bind_parameters(self->statement, parameters, allow_8bit_chars);
+        # Unfortunately there is no way to reach self->statement from Python,
+        # so we quote and substitute parameters manually.
+        if params:
+            if isinstance(params, (list, tuple)):
+                params = self._quote_params_for_last_executed_query(params)
+            else:
+                keys = params.keys()
+                values = tuple(params.values())
+                values = self._quote_params_for_last_executed_query(values)
+                params = dict(zip(keys, values))
+            return sql % params
+        # For consistency with SQLiteCursorWrapper.execute(), just return sql
+        # when there are no parameters. See #13648 and #17158.
+        else:
+            return sql

demo/requirements-demo.pip

@@ -2,3 +2,4 @@ django-cors-headers
 ipdb
 django-extensions
 freezegun
+django-perf-rec

demo/tests/perfs/test_end_point.perf.yml

@@ -0,0 +1,6 @@
+RenderContextSerializer.test_queryset:
+- db: 'SELECT ... FROM "formidable_field" WHERE "formidable_field"."form_id" = # ORDER BY "formidable_field"."order" ASC'
+- db: 'SELECT ... FROM "formidable_access" WHERE ("formidable_access"."access_id" = # AND NOT ("formidable_access"."level" = #) AND "formidable_access"."field_id" IN (...))'
+- db: SELECT ... FROM "formidable_item" WHERE "formidable_item"."field_id" IN (...) ORDER BY "formidable_item"."order" ASC
+- db: SELECT ... FROM "formidable_validation" WHERE "formidable_validation"."field_id" IN (...)
+- db: SELECT ... FROM "formidable_default" WHERE "formidable_default"."field_id" IN (...)

demo/tests/perfs/tests_integration.perf.yml

@@ -0,0 +1,8 @@
+TestContextFormEndPoint.test_queryset:
+- db: 'SELECT ... FROM "django_session" WHERE ("django_session"."expire_date" > # AND "django_session"."session_key" = #)'
+- db: 'SELECT ... FROM "formidable_formidable" WHERE "formidable_formidable"."id" = #'
+- db: 'SELECT ... FROM "formidable_field" WHERE "formidable_field"."form_id" = # ORDER BY "formidable_field"."order" ASC'
+- db: 'SELECT ... FROM "formidable_access" WHERE ("formidable_access"."access_id" = # AND NOT ("formidable_access"."level" = #) AND "formidable_access"."field_id" IN (...))'
+- db: SELECT ... FROM "formidable_item" WHERE "formidable_item"."field_id" IN (...) ORDER BY "formidable_item"."order" ASC
+- db: SELECT ... FROM "formidable_validation" WHERE "formidable_validation"."field_id" IN (...)
+- db: SELECT ... FROM "formidable_default" WHERE "formidable_default"."field_id" IN (...)

demo/tests/test_end_point.py

@@ -5,6 +5,7 @@
 from functools import reduce
 
 from django.test import TestCase
+import django_perf_rec
 
 from formidable import constants
 from formidable.models import Formidable
@@ -298,6 +299,21 @@ class TestForm(FormidableForm):
         defaults = field['defaults']
         self.assertEqual(defaults, ['Roméo'])
 
+    def test_queryset(self):
+
+        class TestForm(FormidableForm):
+            name = fields.CharField(label='Your name', default='Roméo')
+            label = fields.CharField(label='label', default='Roméo')
+            salary = fields.IntegerField()
+            birthdate = fields.DateField()
+
+        form = TestForm.to_formidable(label='title')
+
+        serializer = ContextFormSerializer(form, context={'role': 'jedi'})
+
+        with django_perf_rec.record(path='perfs/'):
+            serializer.data
+
 
 class CreateSerializerTestCase(TestCase):
 

demo/tests/tests_integration.py

@@ -248,6 +248,29 @@ class MyForm(FormidableForm):
     )
 
 
+class TestContextFormEndPoint(APITestCase):
+
+    @classmethod
+    def setUpClass(cls):
+        class MyTestForm(MyForm):
+            phone = fields.IntegerField()
+
+        super(TestContextFormEndPoint, cls).setUpClass()
+        cls.form = MyForm.to_formidable(label='test')
+
+    def test_queryset(self):
+        import django_perf_rec
+
+        session = self.client.session
+        session['role'] = 'padawan'
+        session.save()
+
+        with django_perf_rec.record(path='perfs/'):
+            self.client.get(reverse(
+                'formidable:context_form_detail', args=[self.form.pk])
+            )
+
+
 class TestValidationEndPoint(APITestCase):
 
     def setUp(self):

formidable/serializers/fields.py

@@ -8,7 +8,7 @@
 from rest_framework import serializers
 
 from formidable import constants
-from formidable.models import Access, Field
+from formidable.models import Access, Field, Item
 from formidable.register import FieldSerializerRegister, load_serializer
 from formidable.serializers.access import AccessSerializer
 from formidable.serializers.child_proxy import LazyChildProxy
@@ -95,13 +95,19 @@ def get_attribute(self, instance):
         qs = super(ListContextFieldSerializer, self).get_attribute(instance)
         access_qs = Access.objects.filter(access_id=self.role)
         access_qs = access_qs.exclude(level=constants.HIDDEN)
-        qs = qs.prefetch_related(Prefetch('accesses', queryset=access_qs))
-        return qs
+        qs = qs.prefetch_related(
+            Prefetch('accesses', queryset=access_qs),
+            Prefetch('items', queryset=Item.objects.order_by('order')),
+            'validations', 'defaults',
+        )
+        return qs.order_by('order')
 
     def to_representation(self, fields):
         res = []
-        for field in fields.order_by('order').all():
-            if field.accesses.exists():
+        for field in fields.all():
+            # Avoid to hit the database, the righ access is currently loaded,
+            # unless its an hidden access
+            if field.accesses.count() > 0:
                 res.append(self.child.to_representation(field))
 
         return res
@@ -129,12 +135,24 @@ def role(self):
         return self._context['role']
 
     def get_disabled(self, obj):
-        return obj.accesses.get(access_id=self.role).level == \
-            constants.READONLY
+        # accesses object are already loaded through prefetch inside the
+        # "get_attribute" method, a "get" on related object will
+        # hit the database, a "all" method not.
+        # With the prefetch method and the "exists" check at the
+        # ListContextFieldSerializer.to_representation method, you are sure
+        # to have the access matching the role
+        access = obj.accesses.all()[0]
+        return access.level == constants.READONLY
 
     def get_required(self, obj):
-        return obj.accesses.get(access_id=self.role).level == \
-            constants.REQUIRED
+        # accesses object are already loaded through prefetch inside the
+        # "get_attribute" method, a "get" on related object will
+        # hit the database, a "all" method not.
+        # With the prefetch method and the "exists" check at the
+        # ListContextFieldSerializer.to_representation method, you are sure
+        # to have the access matching the role
+        access = obj.accesses.all()[0]
+        return access.level == constants.REQUIRED
 
 
 class FieldItemMixin(object):

formidable/serializers/items.py

@@ -20,11 +20,6 @@ def validate(self, data):
 
         return data
 
-    def to_representation(self, items):
-        return super(ItemListSerializer, self).to_representation(
-            items.order_by('order')
-        )
-
 
 class ItemSerializer(serializers.ModelSerializer):
 

formidable/urls.py

@@ -4,7 +4,7 @@
 
 urlpatterns = [
     url(r'^forms/(?P<pk>\d+)/$', views.ContextFormDetail.as_view(),
-        name='form_detail'),
+        name='context_form_detail'),
     url(r'^forms/(?P<pk>\d+)/validate/$', views.ValidateView.as_view(),
         name='form_validation'),
     url(r'^builder/forms/(?P<pk>\d+)/$', views.FormidableDetail.as_view(),

formidable/views.py

@@ -197,11 +197,6 @@ class ContextFormDetail(six.with_metaclass(MetaClassView, RetrieveAPIView)):
     serializer_class = ContextFormSerializer
     settings_permission_key = 'FORMIDABLE_PERMISSION_USING'
 
-    def get_queryset(self):
-        qs = super(ContextFormDetail, self).get_queryset()
-        field_qs = Field.objects.order_by('order')
-        return qs.prefetch_related(Prefetch('fields', queryset=field_qs))
-
     def get_serializer_context(self):
         context = super(ContextFormDetail, self).get_serializer_context()
         context['role'] = get_context(self.request, self.kwargs)