#398 Added similar queries count

project/example_app/admin.py

@@ -1,7 +1,7 @@
 from django.contrib import admin
 from django.urls import reverse
 
-from .models import Blind
+from .models import Blind, Category
 
 
 class BlindAdmin(admin.ModelAdmin):
@@ -33,3 +33,4 @@ def desc(self, obj):
 
 
 admin.site.register(Blind, BlindAdmin)
+admin.site.register(Category, admin.ModelAdmin)

project/example_app/migrations/0004_category_blind_category.py

@@ -0,0 +1,39 @@
+# Generated by Django 4.1.3 on 2022-11-19 17:10
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("example_app", "0003_blind_unique_name_if_provided"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="Category",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("name", models.CharField(max_length=50)),
+            ],
+        ),
+        migrations.AddField(
+            model_name="blind",
+            name="category",
+            field=models.ForeignKey(
+                null=True,
+                blank=True,
+                on_delete=django.db.models.deletion.SET_NULL,
+                to="example_app.category",
+            ),
+        ),
+    ]

project/example_app/models.py

@@ -1,19 +1,28 @@
 from django.db import models
+from django.utils.translation import gettext_lazy as _
 
-# Create your models here.
-from django.db.models import BooleanField, ImageField, TextField
+
+class Category(models.Model):
+    name = models.CharField(max_length=50)
+
+    def __str__(self):
+        return self.name
+
+    class Meta:
+        verbose_name_plural = _("Categories")
 
 
 class Product(models.Model):
-    photo = ImageField(upload_to='products')
+    photo = models.ImageField(upload_to='products')
 
     class Meta:
         abstract = True
 
 
 class Blind(Product):
-    name = TextField()
-    child_safe = BooleanField(default=False)
+    name = models.TextField()
+    child_safe = models.BooleanField(default=False)
+    category = models.ForeignKey(Category, on_delete=models.SET_NULL, null=True, blank=True)
 
     def __str__(self):
         return self.name

project/example_app/templates/example_app/index.html

@@ -14,12 +14,14 @@ <h1>Example App</h1>
         <th>Photo</th>
         <th>Name</th>
         <th>Child safe?</th>
+        <th>Category</th>
     </tr>
     {% for blind in blinds %}
         <tr>
             <td>{% if blind.photo %}<img class="blind" src="{{ blind.photo.url }}">{% endif %}</td>
             <td>{{ blind.name }}</td>
             <td>{% if blind.child_safe %}Yes{% else %}No{% endif %}</td>
+            <td>{{ blind.category }}</td>
         </tr>
     {% endfor %}
 </table>

project/example_app/views.py

@@ -21,5 +21,5 @@ def do_something_long():
 
 class ExampleCreateView(CreateView):
     model = models.Blind
-    fields = ['name']
+    fields = ['name', 'category']
     success_url = reverse_lazy('example_app:index')

project/tests/factories.py

@@ -1,6 +1,5 @@
-import factory
 import factory.fuzzy
-from example_app.models import Blind
+from example_app.models import Blind, Category
 
 from silk.models import Request, Response, SQLQuery
 
@@ -34,10 +33,18 @@ class Meta:
         model = Response
 
 
+class CategoryFactory(factory.django.DjangoModelFactory):
+    name = factory.Faker('pystr', min_chars=5, max_chars=10)
+
+    class Meta:
+        model = Category
+
+
 class BlindFactory(factory.django.DjangoModelFactory):
     name = factory.Faker('pystr', min_chars=5, max_chars=10)
     child_safe = factory.Faker('pybool')
     photo = factory.django.ImageField()
+    category = factory.SubFactory(CategoryFactory)
 
     class Meta:
         model = Blind

project/tests/test_lib/mock_suite.py

@@ -82,8 +82,10 @@ def mock_sql_queries(self, request=None, profile=None, n=1, as_dict=False):
         queries = []
         for _ in range(0, n):
             tb = ''.join(reversed(traceback.format_stack()))
+            random_query = self._random_query()
             d = {
-                'query': self._random_query(),
+                'query': random_query,
+                'query_structure': random_query,
                 'start_time': start_time,
                 'end_time': end_time,
                 'request': request,

project/tests/test_view_sql.py

@@ -0,0 +1,28 @@
+from django.test import TestCase
+
+from silk.config import SilkyConfig
+from silk.middleware import silky_reverse
+
+from .test_lib.mock_suite import MockSuite
+
+
+class TestViewSQL(TestCase):
+    @classmethod
+    def setUpClass(cls):
+        super().setUpClass()
+        SilkyConfig().SILKY_AUTHENTICATION = False
+        SilkyConfig().SILKY_AUTHORISATION = False
+
+    def test_duplicates_should_show(self):
+        """Generate a lot of duplicates and test that they are visible on the page"""
+        request = MockSuite().mock_request()
+        request.queries.all().delete()
+        # Ensure we have a amount of queries with the same structure
+        query = MockSuite().mock_sql_queries(request=request, n=1)[0]
+        for _ in range(0, 4):
+            query.id = None
+            query.save()
+        url = silky_reverse('request_sql', kwargs={'request_id': request.id})
+        response = self.client.get(url)
+        self.assertEqual(response.status_code, 200)
+        self.assertContains(response, '<td class="right-aligned">4</td>')

silk/collector.py

@@ -2,6 +2,7 @@
 import logging
 import marshal
 import pstats
+from collections import defaultdict
 from io import StringIO
 from threading import local
 
@@ -149,10 +150,13 @@ def finalise(self):
                 self.request.prof_file = f.name
 
         sql_queries = []
+        duplicate_queries = defaultdict(lambda: -1)
         for identifier, query in self.queries.items():
             query['identifier'] = identifier
             sql_query = models.SQLQuery(**query)
             sql_queries += [sql_query]
+            duplicate_queries[sql_query.query_structure] += 1
+        self.request.num_duplicated_queries = sum(duplicate_queries.values())
 
         models.SQLQuery.objects.bulk_create(sql_queries)
         sql_queries = models.SQLQuery.objects.filter(request=self.request)

silk/migrations/0009_duplicated_queries.py

@@ -0,0 +1,23 @@
+# Generated by Django 3.2.16 on 2022-10-25 17:11
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('silk', '0008_sqlquery_analysis'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='request',
+            name='num_duplicated_queries',
+            field=models.IntegerField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name='sqlquery',
+            name='query_structure',
+            field=models.TextField(default=''),
+        ),
+    ]

silk/models.py

@@ -75,6 +75,7 @@ class Request(models.Model):
     meta_time = FloatField(null=True, blank=True)
     meta_num_queries = IntegerField(null=True, blank=True)
     meta_time_spent_queries = FloatField(null=True, blank=True)
+    num_duplicated_queries = IntegerField(null=True, blank=True)
     pyprofile = TextField(blank=True, default='')
     prof_file = FileField(max_length=300, blank=True, storage=silk_storage)
 
@@ -237,6 +238,7 @@ def bulk_create(self, *args, **kwargs):
 
 class SQLQuery(models.Model):
     query = TextField()
+    query_structure = TextField(default='')
     start_time = DateTimeField(null=True, blank=True, default=timezone.now)
     end_time = DateTimeField(null=True, blank=True)
     time_taken = FloatField(blank=True, null=True)

silk/sql.py

@@ -82,6 +82,7 @@ def execute_sql(self, *args, **kwargs):
     if _should_wrap(sql_query):
         query_dict = {
             'query': sql_query,
+            'query_structure': q,
             'start_time': timezone.now(),
             'traceback': tb
         }

silk/templates/silk/inclusion/request_summary.html

@@ -12,6 +12,12 @@
         <span class="appenage">on queries<span class="meta">{% if silk_request.meta_time_spent_queries %} +{{ silk_request.meta_time_spent_queries | floatformat:"0" }}<span class="unit">ms</span>{% endif %}</span></div>
     <div class="num-queries-div">
         <span class="numeric">{{ silk_request.num_sql_queries }}</span>
-        <span class="appendage">queries<span class="meta">{% if silk_request.meta_num_queries %} +{{ silk_request.meta_num_queries  }}{% endif %}</span>
+        <span class="appendage">queries{% if silk_request.meta_num_queries %}<span class="meta"> +{{ silk_request.meta_num_queries }}</span>{% endif %}</span>
     </div>
+    {% if silk_request.num_duplicated_queries %}
+    <div class="num-duplicated-queries-div">
+        <span class="numeric">{{ silk_request.num_duplicated_queries }}</span>
+        <span class="appendage">duplicated queries</span>
+    </div>
+    {% endif %}
 </div>

silk/templates/silk/inclusion/request_summary_row.html

@@ -8,8 +8,15 @@
     </div>
     <div class="col time-taken-queries-div spacing">
         <span class="col numeric">{{ silk_request.time_spent_on_sql_queries|floatformat:"0" }}<span class="unit">ms</span></span>
-        <span class="col appenage">on queries<span class="meta">{% if silk_request.meta_time_spent_queries %} +{{ silk_request.meta_time_spent_queries | floatformat:"0" }}<span class="unit">ms</span>{% endif %}</span></div>
+        <span class="col appenage">on queries<span class="meta">{% if silk_request.meta_time_spent_queries %} +{{ silk_request.meta_time_spent_queries | floatformat:"0" }}<span class="unit">ms</span>{% endif %}</span></span>
+    </div>
     <div class="col num-queries-div spacing">
         <span class="col numeric">{{ silk_request.num_sql_queries }}</span>
-        <span class="col appendage">queries<span class="meta">{% if silk_request.meta_num_queries %} +{{ silk_request.meta_num_queries  }}{% endif %}</span></span>
+        <span class="col appendage">queries{% if silk_request.meta_num_queries %}<span class="meta"> +{{ silk_request.meta_num_queries }}</span>{% endif %}</span>
+    </div>
+    {% if silk_request.num_duplicated_queries %}
+    <div class="col num-duplicated-queries-div spacing">
+        <span class="col numeric">{{ silk_request.num_duplicated_queries }}</span>
+        <span class="col appendage">duplicated queries</span>
     </div>
+    {% endif %}

silk/templates/silk/sql.html

@@ -41,7 +41,7 @@
             height: 20px;
         }
 
-        tr.data-row:hover {
+        tr.data-row:hover, tr.data-row.highlight {
             background-color: rgb(51, 51, 68);
             color: white;
             cursor: pointer;
@@ -109,10 +109,14 @@
                     <th class="left-aligned">Tables</th>
                     <th class="right-aligned">Num. Joins</th>
                     <th class="right-aligned">Execution Time (ms)</th>
+                    <th class="right-aligned">Num. Duplicates</th>
                 </tr>
                 {% for sql_query in items %}
                     <!-- TODO: Pretty grimy... -->
-                    <tr class="data-row" onclick="window.location=' \
+                    <tr
+                        class="data-row"
+                        data-duplicate-id="{{ sql_query.duplicate_id }}"
+                        onclick="window.location=' \
                        {% if profile and silk_request %}\
                            {% url "silk:request_and_profile_sql_detail" silk_request.id profile.id sql_query.id %}\
                        {% elif profile %}\
@@ -126,6 +130,7 @@
                         <td class="left-aligned">{{ sql_query.tables_involved|join:", " }}</td>
                         <td class="right-aligned">{{ sql_query.num_joins }}</td>
                         <td class="right-aligned">{{ sql_query.time_taken | floatformat:6 }}</td>
+                        <td class="right-aligned">{{ sql_query.num_duplicates }}</td>
                     </tr>
                 {% endfor %}
 
@@ -153,8 +158,20 @@
         </div>
     </div>
 
-
-
+<script>
+    const rows = document.querySelectorAll('tr.data-row').forEach(function (row) {
+        row.addEventListener('mouseenter', function () {
+            document.querySelectorAll('tr[data-duplicate-id="' + row.dataset.duplicateId + '"]').forEach(function (e) {
+                e.classList.add('highlight');
+            });
+        });
+        row.addEventListener('mouseleave', function () {
+            document.querySelectorAll('tr[data-duplicate-id="' + row.dataset.duplicateId + '"]').forEach(function (e) {
+                e.classList.remove('highlight');
+            });
+        });
+    });
+</script>
 
 
 {% endblock %}

silk/views/sql.py

@@ -1,3 +1,5 @@
+from collections import defaultdict
+
 from django.shortcuts import render
 from django.utils.decorators import method_decorator
 from django.views.generic import View
@@ -20,10 +22,17 @@ def get(self, request, *_, **kwargs):
             'request': request,
         }
         if request_id:
+            duplicate_queries = defaultdict(lambda: -1)
             silk_request = Request.objects.get(id=request_id)
             query_set = SQLQuery.objects.filter(request=silk_request).order_by('-start_time')
             for q in query_set:
                 q.start_time_relative = q.start_time - silk_request.start_time
+                duplicate_queries[q.query_structure] += 1
+            structures = list(duplicate_queries.keys())
+            for q in query_set:
+                if q.query_structure:
+                    q.num_duplicates = duplicate_queries[q.query_structure]
+                    q.duplicate_id = structures.index(q.query_structure)
             page = _page(request, query_set)
             context['silk_request'] = silk_request
         if profile_id: