Depublicate queries retrieve builder view

demo/demo/settings.py

@@ -12,6 +12,7 @@
 
 # Build paths inside the project like this: os.path.join(BASE_DIR, ...)
 import os
+import django
 
 BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 
@@ -80,9 +81,14 @@
 # Database
 # https://docs.djangoproject.com/en/1.8/ref/settings/#databases
 
+if django.VERSION[:2] == (1, 8):
+    engine = 'django18_sqlite3_backend'
+else:
+    engine = 'django.db.backends.sqlite3',
+
 DATABASES = {
     'default': {
-        'ENGINE': 'django.db.backends.sqlite3',
+        'ENGINE': engine,
         'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
     }
 }

demo/django18_sqlite3_backend/__init__.py

@@ -0,0 +1,3 @@
+# -*- coding:utf-8 -*-
+# flake8: noqa
+from __future__ import absolute_import, division, print_function, unicode_literals

demo/django18_sqlite3_backend/base.py

@@ -0,0 +1,14 @@
+# -*- coding:utf-8 -*-
+# flake8: noqa
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+from django.db.backends.sqlite3.base import DatabaseWrapper as OrigDatabaseWrapper
+
+from .operations import DatabaseOperations
+
+
+class DatabaseWrapper(OrigDatabaseWrapper):
+
+    def __init__(self, *args, **kwargs):
+        super(DatabaseWrapper, self).__init__(*args, **kwargs)
+        self.ops = DatabaseOperations(self)

demo/django18_sqlite3_backend/operations.py

@@ -0,0 +1,43 @@
+# -*- coding:utf-8 -*-
+# flake8: noqa
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+from django.db.backends.sqlite3.operations import DatabaseOperations as OrigDatabaseOperations
+
+
+class DatabaseOperations(OrigDatabaseOperations):
+
+    # From Django commit 4f6a7663bcddffb114f2647f9928cbf1fdd8e4b5
+
+    def _quote_params_for_last_executed_query(self, params):
+        """
+        Only for last_executed_query! Don't use this to execute SQL queries!
+        """
+        sql = 'SELECT ' + ', '.join(['QUOTE(?)'] * len(params))
+        # Bypass Django's wrappers and use the underlying sqlite3 connection
+        # to avoid logging this query - it would trigger infinite recursion.
+        cursor = self.connection.connection.cursor()
+        # Native sqlite3 cursors cannot be used as context managers.
+        try:
+            return cursor.execute(sql, params).fetchone()
+        finally:
+            cursor.close()
+
+    def last_executed_query(self, cursor, sql, params):
+        # Python substitutes parameters in Modules/_sqlite/cursor.c with:
+        # pysqlite_statement_bind_parameters(self->statement, parameters, allow_8bit_chars);
+        # Unfortunately there is no way to reach self->statement from Python,
+        # so we quote and substitute parameters manually.
+        if params:
+            if isinstance(params, (list, tuple)):
+                params = self._quote_params_for_last_executed_query(params)
+            else:
+                keys = params.keys()
+                values = tuple(params.values())
+                values = self._quote_params_for_last_executed_query(values)
+                params = dict(zip(keys, values))
+            return sql % params
+        # For consistency with SQLiteCursorWrapper.execute(), just return sql
+        # when there are no parameters. See #13648 and #17158.
+        else:
+            return sql

demo/requirements-demo.pip

@@ -2,3 +2,4 @@ django-cors-headers
 ipdb
 django-extensions
 freezegun
+django-perf-rec

demo/tests/perfs/test_end_point.perf.yml

@@ -0,0 +1,13 @@
+RenderContextSerializer.test_queryset:
+- db: 'SELECT ... FROM "formidable_field" WHERE "formidable_field"."form_id" = # ORDER BY "formidable_field"."order" ASC'
+- db: 'SELECT ... FROM "formidable_access" WHERE ("formidable_access"."access_id" = # AND NOT ("formidable_access"."level" = #) AND "formidable_access"."field_id" IN (...))'
+- db: SELECT ... FROM "formidable_item" WHERE "formidable_item"."field_id" IN (...) ORDER BY "formidable_item"."order" ASC
+- db: SELECT ... FROM "formidable_validation" WHERE "formidable_validation"."field_id" IN (...)
+- db: SELECT ... FROM "formidable_default" WHERE "formidable_default"."field_id" IN (...)
+RenderSerializerTestCase.test_queryset:
+- db: 'SELECT ... FROM "formidable_field" WHERE "formidable_field"."form_id" = # ORDER BY "formidable_field"."order" ASC'
+- db: SELECT ... FROM "formidable_item" WHERE "formidable_item"."field_id" IN (...) ORDER BY "formidable_item"."order" ASC
+- db: SELECT ... FROM "formidable_default" WHERE "formidable_default"."field_id" IN (...)
+- db: SELECT ... FROM "formidable_validation" WHERE "formidable_validation"."field_id" IN (...)
+- db: SELECT ... FROM "formidable_access" WHERE "formidable_access"."field_id" IN (...)
+- db: 'SELECT ... FROM "formidable_preset" WHERE "formidable_preset"."form_id" = #'

demo/tests/perfs/tests_integration.perf.yml

@@ -0,0 +1,17 @@
+TestContextFormEndPoint.test_queryset:
+- db: 'SELECT ... FROM "django_session" WHERE ("django_session"."expire_date" > # AND "django_session"."session_key" = #)'
+- db: 'SELECT ... FROM "formidable_formidable" WHERE "formidable_formidable"."id" = #'
+- db: 'SELECT ... FROM "formidable_field" WHERE "formidable_field"."form_id" = # ORDER BY "formidable_field"."order" ASC'
+- db: 'SELECT ... FROM "formidable_access" WHERE ("formidable_access"."access_id" = # AND NOT ("formidable_access"."level" = #) AND "formidable_access"."field_id" IN (...))'
+- db: SELECT ... FROM "formidable_item" WHERE "formidable_item"."field_id" IN (...) ORDER BY "formidable_item"."order" ASC
+- db: SELECT ... FROM "formidable_validation" WHERE "formidable_validation"."field_id" IN (...)
+- db: SELECT ... FROM "formidable_default" WHERE "formidable_default"."field_id" IN (...)
+UpdateFormTestCase.test_queryset_on_get:
+- db: 'SELECT ... FROM "formidable_formidable" WHERE "formidable_formidable"."id" = #'
+- db: SELECT ... FROM "formidable_field" WHERE "formidable_field"."form_id" IN (#) ORDER BY "formidable_field"."order" ASC
+- db: 'SELECT ... FROM "formidable_field" WHERE "formidable_field"."form_id" = # ORDER BY "formidable_field"."order" ASC'
+- db: SELECT ... FROM "formidable_item" WHERE "formidable_item"."field_id" IN (...) ORDER BY "formidable_item"."order" ASC
+- db: SELECT ... FROM "formidable_default" WHERE "formidable_default"."field_id" IN (...)
+- db: SELECT ... FROM "formidable_validation" WHERE "formidable_validation"."field_id" IN (...)
+- db: SELECT ... FROM "formidable_access" WHERE "formidable_access"."field_id" IN (...)
+- db: 'SELECT ... FROM "formidable_preset" WHERE "formidable_preset"."form_id" = #'

demo/tests/test_end_point.py

@@ -5,6 +5,7 @@
 from functools import reduce
 
 from django.test import TestCase
+import django_perf_rec
 
 from formidable import constants
 from formidable.models import Formidable
@@ -195,6 +196,20 @@ def test_separator_field(self):
         self.assertNotIn('label', data)
         self.assertNotIn('help_text', data)
 
+    def test_queryset(self):
+        class MyTestForm(FormidableForm):
+            first_name = fields.CharField(default='foo')
+            last_name = fields.CharField()
+            origin = fields.ChoiceField(choices=(
+                ('fr', 'France'),
+                ('us', 'United States'),
+            ))
+
+        formidable = MyTestForm.to_formidable(label='test')
+        serializer = FormidableSerializer(instance=formidable)
+        with django_perf_rec.record(path='perfs/'):
+            serializer.data
+
 
 class RenderContextSerializer(TestCase):
 
@@ -298,6 +313,21 @@ class TestForm(FormidableForm):
         defaults = field['defaults']
         self.assertEqual(defaults, ['Roméo'])
 
+    def test_queryset(self):
+
+        class TestForm(FormidableForm):
+            name = fields.CharField(label='Your name', default='Roméo')
+            label = fields.CharField(label='label', default='Roméo')
+            salary = fields.IntegerField()
+            birthdate = fields.DateField()
+
+        form = TestForm.to_formidable(label='title')
+
+        serializer = ContextFormSerializer(form, context={'role': 'jedi'})
+
+        with django_perf_rec.record(path='perfs/'):
+            serializer.data
+
 
 class CreateSerializerTestCase(TestCase):
 

demo/tests/tests_integration.py

@@ -5,6 +5,7 @@
 from copy import deepcopy
 
 from django.core.urlresolvers import reverse
+import django_perf_rec
 
 from freezegun import freeze_time
 from rest_framework.test import APITestCase
@@ -17,6 +18,33 @@
 from . import form_data, form_data_items
 
 
+class MyTestForm(FormidableForm):
+
+    name = fields.CharField(label='Name', accesses={'jedi': 'REQUIRED'})
+    birth_date = fields.DateField(
+        label='Your Birth Date',
+        validators=[
+            validators.AgeAboveValidator(
+                21, message='You cannot be a jedi until your 21'
+            ),
+            validators.DateIsInFuture(False)
+        ],
+        accesses={'jedi': 'REQUIRED'},
+    )
+    out_date = fields.DateField(
+        validators=[validators.DateIsInFuture(True)]
+    )
+    weapons = fields.ChoiceField(choices=[
+        ('gun', 'blaster'), ('sword', 'light saber')
+    ])
+    salary = fields.IntegerField(
+        validators=[
+            validators.GTValidator(0), validators.LTEValidator(25)
+        ],
+        accesses={'jedi': 'HIDDEN', 'jedi-master': 'REQUIRED'}
+    )
+
+
 class CreateFormTestCase(APITestCase):
 
     def test_simple(self):
@@ -73,6 +101,11 @@ def test_with_unknown_accesses(self):
 
 class UpdateFormTestCase(APITestCase):
 
+    @classmethod
+    def setUpClass(cls):
+        super(UpdateFormTestCase, cls).setUpClass()
+        cls.formidable_form = MyTestForm.to_formidable(label='test')
+
     def setUp(self):
         super(UpdateFormTestCase, self).setUp()
         self.form = Formidable.objects.create(
@@ -164,6 +197,12 @@ def test_delete_field_on_update(self):
             access_id="jedi-master", level="READONLY"
         ).exists())
 
+    def test_queryset_on_get(self):
+        with django_perf_rec.record(path='perfs/'):
+            self.client.get(reverse(
+                'formidable:form_detail', args=[self.formidable_form.pk])
+            )
+
 
 class TestAccess(APITestCase):
 
@@ -184,34 +223,9 @@ def test_get(self):
 
 class TestChain(APITestCase):
 
-    class MyTestForm(FormidableForm):
-
-        name = fields.CharField(label='Name', accesses={'jedi': 'REQUIRED'})
-        birth_date = fields.DateField(
-            label='Your Birth Date', validators=[
-                validators.AgeAboveValidator(
-                    21, message='You cannot be a jedi until your 21'
-                ),
-                validators.DateIsInFuture(False)
-            ],
-            accesses={'jedi': 'REQUIRED'},
-        )
-        out_date = fields.DateField(
-            validators=[validators.DateIsInFuture(True)]
-        )
-        weapons = fields.ChoiceField(choices=[
-            ('gun', 'blaster'), ('sword', 'light saber')
-        ])
-        salary = fields.IntegerField(
-            validators=[
-                validators.GTValidator(0), validators.LTEValidator(25)
-            ],
-            accesses={'jedi': 'HIDDEN', 'jedi-master': 'REQUIRED'}
-        )
-
     def setUp(self):
         super(APITestCase, self).setUp()
-        self.form = self.MyTestForm.to_formidable(label='Jedi Form')
+        self.form = MyTestForm.to_formidable(label='Jedi Form')
         self.assertTrue(self.form.pk)
 
     @freeze_time('2021-01-01')
@@ -248,6 +262,29 @@ class MyForm(FormidableForm):
     )
 
 
+class TestContextFormEndPoint(APITestCase):
+
+    @classmethod
+    def setUpClass(cls):
+        class MyTestForm(MyForm):
+            phone = fields.IntegerField()
+
+        super(TestContextFormEndPoint, cls).setUpClass()
+        cls.form = MyForm.to_formidable(label='test')
+
+    def test_queryset(self):
+        import django_perf_rec
+
+        session = self.client.session
+        session['role'] = 'padawan'
+        session.save()
+
+        with django_perf_rec.record(path='perfs/'):
+            self.client.get(reverse(
+                'formidable:context_form_detail', args=[self.form.pk])
+            )
+
+
 class TestValidationEndPoint(APITestCase):
 
     def setUp(self):

formidable/serializers/fields.py

@@ -8,7 +8,7 @@
 from rest_framework import serializers
 
 from formidable import constants
-from formidable.models import Access, Field
+from formidable.models import Access, Field, Item
 from formidable.register import FieldSerializerRegister, load_serializer
 from formidable.serializers.access import AccessSerializer
 from formidable.serializers.child_proxy import LazyChildProxy
@@ -47,6 +47,14 @@ def validate(self, validated_data):
 
         return validated_data
 
+    def get_attribute(self, instance):
+        qs = super(FieldListSerializer, self).get_attribute(instance)
+        qs = qs.prefetch_related(
+            Prefetch('items', queryset=Item.objects.order_by('order')),
+            'defaults', 'validations', 'accesses'
+        )
+        return qs.order_by('order')
+
 
 class FieldSerializer(WithNestedSerializer):
 
@@ -95,13 +103,19 @@ def get_attribute(self, instance):
         qs = super(ListContextFieldSerializer, self).get_attribute(instance)
         access_qs = Access.objects.filter(access_id=self.role)
         access_qs = access_qs.exclude(level=constants.HIDDEN)
-        qs = qs.prefetch_related(Prefetch('accesses', queryset=access_qs))
-        return qs
+        qs = qs.prefetch_related(
+            Prefetch('accesses', queryset=access_qs),
+            Prefetch('items', queryset=Item.objects.order_by('order')),
+            'validations', 'defaults',
+        )
+        return qs.order_by('order')
 
     def to_representation(self, fields):
         res = []
-        for field in fields.order_by('order').all():
-            if field.accesses.exists():
+        for field in fields.all():
+            # Avoid to hit the database, the righ access is currently loaded,
+            # unless its an hidden access
+            if field.accesses.count() > 0:
                 res.append(self.child.to_representation(field))
 
         return res
@@ -129,12 +143,24 @@ def role(self):
         return self._context['role']
 
     def get_disabled(self, obj):
-        return obj.accesses.get(access_id=self.role).level == \
-            constants.READONLY
+        # accesses object are already loaded through prefetch inside the
+        # "get_attribute" method, a "get" on related object will
+        # hit the database, a "all" method not.
+        # With the prefetch method and the "exists" check at the
+        # ListContextFieldSerializer.to_representation method, you are sure
+        # to have the access matching the role
+        access = obj.accesses.all()[0]
+        return access.level == constants.READONLY
 
     def get_required(self, obj):
-        return obj.accesses.get(access_id=self.role).level == \
-            constants.REQUIRED
+        # accesses object are already loaded through prefetch inside the
+        # "get_attribute" method, a "get" on related object will
+        # hit the database, a "all" method not.
+        # With the prefetch method and the "exists" check at the
+        # ListContextFieldSerializer.to_representation method, you are sure
+        # to have the access matching the role
+        access = obj.accesses.all()[0]
+        return access.level == constants.REQUIRED
 
 
 class FieldItemMixin(object):

formidable/serializers/items.py

@@ -20,11 +20,6 @@ def validate(self, data):
 
         return data
 
-    def to_representation(self, items):
-        return super(ItemListSerializer, self).to_representation(
-            items.order_by('order')
-        )
-
 
 class ItemSerializer(serializers.ModelSerializer):