From fbdd2ebcd4691067fb3a9f0fa61298a132cab676 Mon Sep 17 00:00:00 2001 From: Samson Nkrumah Date: Mon, 20 Apr 2026 21:47:12 +0100 Subject: [PATCH 1/9] feat: add duration/participants/issues summary endpoints (Phase 2 + 3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Five new endpoints for the remaining dashboard charts that fetch raw data: - GET /v1/conferences/duration-summary Returns conference counts bucketed by duration range (< 1m, 1-3m, etc.) - GET /v1/conferences/participant-count-summary Returns distribution of conferences by participant count - GET /v1/issues/summary Returns issue counts grouped by code with titles - GET /v1/issues/gum-summary Returns getusermedia_error issue counts grouped by error name Also adds three new filter params to /v1/conferences for click-to-detail modals on these charts: - duration_gte, duration_lt (for duration chart) - issue_code (for most-common-issues chart) All endpoints accept appId, created_at_gte, created_at_lte and handle both Python native ISO format and JavaScript's toISOString Z suffix. Phases 2 and 3 of #20 — eliminates the need for the dashboard to download all conferences (~38MB) and all issues (~73MB). --- app/urls.py | 7 + app/views/conference_duration_summary_view.py | 90 ++++++++++++ ...nference_participant_count_summary_view.py | 77 +++++++++++ app/views/conference_view.py | 3 + app/views/issue_summary_view.py | 128 ++++++++++++++++++ 5 files changed, 305 insertions(+) create mode 100644 app/views/conference_duration_summary_view.py create mode 100644 app/views/conference_participant_count_summary_view.py create mode 100644 app/views/issue_summary_view.py diff --git a/app/urls.py b/app/urls.py index f1fbf25..e95228c 100644 --- a/app/urls.py +++ b/app/urls.py @@ -6,6 +6,9 @@ from .views.browser_event_view import BrowserEventView from .views.conference_view import ConferencesView from .views.conference_summary_view import ConferenceSummaryView +from .views.conference_duration_summary_view import ConferenceDurationSummaryView +from .views.conference_participant_count_summary_view import ConferenceParticipantCountSummaryView +from .views.issue_summary_view import IssueSummaryView, GetUserMediaSummaryView from .views.connection_event_view import ConnectionEventView from .views.connection_view import ConnectionView from .views.issue_view import IssueView @@ -39,6 +42,8 @@ path('connections', ConnectionView.as_view(), name='connections'), path('connections/', ConnectionView.as_view(), name='connection'), path('issues', IssueView.as_view(), name='issues'), + path('issues/summary', IssueSummaryView.as_view(), name='issues-summary'), + path('issues/gum-summary', GetUserMediaSummaryView.as_view(), name='issues-gum-summary'), path('issues/', IssueView.as_view(), name='issue'), path('stats', StatsView.as_view(), name='stats'), @@ -57,6 +62,8 @@ path('conferences', ConferencesView.as_view(), name='conferences'), path('conferences/summary', ConferenceSummaryView.as_view(), name='conferences-summary'), + path('conferences/duration-summary', ConferenceDurationSummaryView.as_view(), name='conferences-duration-summary'), + path('conferences/participant-count-summary', ConferenceParticipantCountSummaryView.as_view(), name='conferences-participant-count-summary'), path('conferences/', ConferencesView.as_view(), name='conference'), path('conferences//events', ConferenceEventsView.as_view(), name='conference-events'), path('conferences//graphs', ConferenceGraphView.as_view(), name='conference-graphs'), diff --git a/app/views/conference_duration_summary_view.py b/app/views/conference_duration_summary_view.py new file mode 100644 index 0000000..b12e312 --- /dev/null +++ b/app/views/conference_duration_summary_view.py @@ -0,0 +1,90 @@ +import datetime + +from django.core.exceptions import ValidationError +from django.db.models import Case, Count, IntegerField, When + +from ..errors import INVALID_PARAMETERS, MISSING_PARAMETERS, PMError +from ..utils import JSONHttpResponse +from ..models.conference import Conference +from .generic_view import GenericView + + +BUCKETS = [ + {'title': '< 1 m', 'min_sec': 0, 'max_sec': 60}, + {'title': '1 - 3 m', 'min_sec': 60, 'max_sec': 180}, + {'title': '3 - 5 m', 'min_sec': 180, 'max_sec': 300}, + {'title': '5 - 10 m', 'min_sec': 300, 'max_sec': 600}, + {'title': '10 - 15 m','min_sec': 600, 'max_sec': 900}, + {'title': '15 - 20 m','min_sec': 900, 'max_sec': 1200}, + {'title': '20 - 25 m','min_sec': 1200, 'max_sec': 1500}, + {'title': '25 - 30 m','min_sec': 1500, 'max_sec': 1800}, + {'title': '30 - 40 m','min_sec': 1800, 'max_sec': 2400}, + {'title': '40 - 50 m','min_sec': 2400, 'max_sec': 3000}, + {'title': '50 - 60 m','min_sec': 3000, 'max_sec': 3600}, + {'title': '> 60 m', 'min_sec': 3600, 'max_sec': None}, +] + + +def _parse_iso(value): + if value.endswith('Z'): + value = value[:-1] + '+00:00' + return datetime.datetime.fromisoformat(value) + + +class ConferenceDurationSummaryView(GenericView): + """Returns conference count bucketed by duration range.""" + + @classmethod + def get(cls, request): + app_id = request.GET.get('appId') + if not app_id: + raise PMError(status=400, app_error=MISSING_PARAMETERS) + + filters = {'app_id': app_id, 'is_active': True} + + created_at_gte = request.GET.get('created_at_gte') + if created_at_gte: + try: + filters['created_at__gte'] = _parse_iso(created_at_gte) + except ValueError: + raise PMError(status=400, app_error=INVALID_PARAMETERS) + + created_at_lte = request.GET.get('created_at_lte') + if created_at_lte: + try: + filters['created_at__lte'] = _parse_iso(created_at_lte) + except ValueError: + raise PMError(status=400, app_error=INVALID_PARAMETERS) + + whens = [] + for i, b in enumerate(BUCKETS): + lo, hi = b['min_sec'], b['max_sec'] + if hi is None: + whens.append(When(duration__gte=lo, then=i)) + elif lo == 0: + whens.append(When(duration__lt=hi, then=i)) + else: + whens.append(When(duration__gte=lo, duration__lt=hi, then=i)) + + try: + rows = (Conference.objects + .filter(**filters) + .annotate(bucket=Case(*whens, output_field=IntegerField())) + .values('bucket') + .annotate(count=Count('id')) + .order_by('bucket')) + except ValidationError: + raise PMError(status=400, app_error=INVALID_PARAMETERS) + + counts_by_bucket = {r['bucket']: r['count'] for r in rows if r['bucket'] is not None} + data = [ + { + 'range': b['title'], + 'min_sec': b['min_sec'], + 'max_sec': b['max_sec'], + 'count': counts_by_bucket.get(i, 0), + } + for i, b in enumerate(BUCKETS) + ] + + return JSONHttpResponse({'data': data}) diff --git a/app/views/conference_participant_count_summary_view.py b/app/views/conference_participant_count_summary_view.py new file mode 100644 index 0000000..a5bf7ec --- /dev/null +++ b/app/views/conference_participant_count_summary_view.py @@ -0,0 +1,77 @@ +import datetime +from collections import Counter + +from django.core.exceptions import ValidationError +from django.db.models import Count, IntegerField, OuterRef, Subquery + +from ..errors import INVALID_PARAMETERS, MISSING_PARAMETERS, PMError +from ..utils import JSONHttpResponse +from ..models.conference import Conference +from ..models.participant import Participant +from .generic_view import GenericView + + +def _parse_iso(value): + if value.endswith('Z'): + value = value[:-1] + '+00:00' + return datetime.datetime.fromisoformat(value) + + +class ConferenceParticipantCountSummaryView(GenericView): + """ + Returns the distribution of conferences grouped by how many participants + they had. Used by the Number-of-participants pie chart. + """ + + @classmethod + def get(cls, request): + app_id = request.GET.get('appId') + if not app_id: + raise PMError(status=400, app_error=MISSING_PARAMETERS) + + filters = {'app_id': app_id, 'is_active': True} + + created_at_gte = request.GET.get('created_at_gte') + if created_at_gte: + try: + filters['created_at__gte'] = _parse_iso(created_at_gte) + except ValueError: + raise PMError(status=400, app_error=INVALID_PARAMETERS) + + created_at_lte = request.GET.get('created_at_lte') + if created_at_lte: + try: + filters['created_at__lte'] = _parse_iso(created_at_lte) + except ValueError: + raise PMError(status=400, app_error=INVALID_PARAMETERS) + + try: + per_conf = (Conference.objects + .filter(**filters) + .annotate( + participants_count=Subquery( + Participant.objects.filter( + conferences=OuterRef('pk'), + is_active=True, + ).order_by().values('conferences').annotate( + cnt=Count('id', distinct=True) + ).values('cnt')[:1], + output_field=IntegerField(), + ), + ) + .values_list('participants_count', flat=True)) + except ValidationError: + raise PMError(status=400, app_error=INVALID_PARAMETERS) + + distribution = Counter() + total = 0 + for count in per_conf: + distribution[count or 0] += 1 + total += 1 + + data = [ + {'participants': n, 'conferences': c} + for n, c in sorted(distribution.items()) + ] + + return JSONHttpResponse({'data': data, 'total_conferences': total}) diff --git a/app/views/conference_view.py b/app/views/conference_view.py index 9d35088..f08e7d2 100644 --- a/app/views/conference_view.py +++ b/app/views/conference_view.py @@ -30,6 +30,9 @@ def filter(cls, request): 'created_at__gte': 'created_at_gte', 'created_at__lt': 'created_at_lt', 'created_at__lte': 'created_at_lte', + 'duration__gte': 'duration_gte', + 'duration__lt': 'duration_lt', + 'issues__code': 'issue_code', } for key, rkey in allowed_filters.items(): diff --git a/app/views/issue_summary_view.py b/app/views/issue_summary_view.py new file mode 100644 index 0000000..b850e59 --- /dev/null +++ b/app/views/issue_summary_view.py @@ -0,0 +1,128 @@ +import datetime + +from django.core.exceptions import ValidationError +from django.db.models import Count + +from ..errors import INVALID_PARAMETERS, MISSING_PARAMETERS, PMError +from ..utils import JSONHttpResponse +from ..models.issue import Issue, ISSUES +from .generic_view import GenericView + + +def _parse_iso(value): + if value.endswith('Z'): + value = value[:-1] + '+00:00' + return datetime.datetime.fromisoformat(value) + + +class IssueSummaryView(GenericView): + """ + Returns issue counts grouped by code. Used by the Most-common-issues chart. + Replaces downloading all issues (73 MB+ on production) to aggregate in + the browser. + """ + + @classmethod + def get(cls, request): + app_id = request.GET.get('appId') + if not app_id: + raise PMError(status=400, app_error=MISSING_PARAMETERS) + + filters = { + 'conference__app_id': app_id, + 'is_active': True, + } + + created_at_gte = request.GET.get('created_at_gte') + if created_at_gte: + try: + filters['created_at__gte'] = _parse_iso(created_at_gte) + except ValueError: + raise PMError(status=400, app_error=INVALID_PARAMETERS) + + created_at_lte = request.GET.get('created_at_lte') + if created_at_lte: + try: + filters['created_at__lte'] = _parse_iso(created_at_lte) + except ValueError: + raise PMError(status=400, app_error=INVALID_PARAMETERS) + + try: + rows = (Issue.objects + .filter(**filters) + .values('code') + .annotate(count=Count('id')) + .order_by('-count')) + except ValidationError: + raise PMError(status=400, app_error=INVALID_PARAMETERS) + + data = [ + { + 'code': r['code'], + 'title': ISSUES.get(r['code'], {}).get('title', r['code']), + 'count': r['count'], + } + for r in rows + ] + + return JSONHttpResponse({'data': data}) + + +class GetUserMediaSummaryView(GenericView): + """ + Returns counts of getusermedia_error issues grouped by the error's `name` + field inside the JSON data. Used by the GUM (getUserMedia errors) chart. + """ + + @classmethod + def get(cls, request): + app_id = request.GET.get('appId') + if not app_id: + raise PMError(status=400, app_error=MISSING_PARAMETERS) + + filters = { + 'conference__app_id': app_id, + 'code': 'getusermedia_error', + 'is_active': True, + } + + created_at_gte = request.GET.get('created_at_gte') + if created_at_gte: + try: + filters['created_at__gte'] = _parse_iso(created_at_gte) + except ValueError: + raise PMError(status=400, app_error=INVALID_PARAMETERS) + + created_at_lte = request.GET.get('created_at_lte') + if created_at_lte: + try: + filters['created_at__lte'] = _parse_iso(created_at_lte) + except ValueError: + raise PMError(status=400, app_error=INVALID_PARAMETERS) + + from collections import Counter + name_counter = Counter() + message_by_name = {} + + try: + issues = Issue.objects.filter(**filters).values_list('data', flat=True) + for data in issues: + if not data: + continue + name = data.get('name') or 'Unknown' + name_counter[name] += 1 + if name not in message_by_name and data.get('message'): + message_by_name[name] = data['message'] + except ValidationError: + raise PMError(status=400, app_error=INVALID_PARAMETERS) + + data = [ + { + 'name': name, + 'message': message_by_name.get(name, ''), + 'count': count, + } + for name, count in name_counter.most_common() + ] + + return JSONHttpResponse({'data': data, 'total': sum(name_counter.values())}) From 7cd2ec456c76ba08cf5860705d9a021829511dd9 Mon Sep 17 00:00:00 2001 From: Samson Nkrumah Date: Mon, 20 Apr 2026 22:20:34 +0100 Subject: [PATCH 2/9] feat: add connections + sessions summary endpoints (Phase 4 + 5 of #20) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds three new aggregation endpoints that let the dashboard stop downloading full /connections and /sessions payloads to build charts client-side: - GET /v1/connections/summary — relay vs direct connection counts (replaces the Relayed-connections pie chart's client-side reduce) - GET /v1/connections/setup-time-summary — connection setup-time buckets with per-bucket conference_ids for click-to-detail - GET /v1/sessions/summary — browsers, OS, country, and city/geo aggregates (powers Browsers, OS, and Map charts in one roundtrip) Also accepts `conference_ids=a,b,c` on /conferences so the setup-time chart can page through matched conferences on click. Co-Authored-By: Claude Opus 4.7 (1M context) --- app/urls.py | 5 + app/views/conference_view.py | 6 + app/views/connection_summary_view.py | 187 +++++++++++++++++++++++++++ app/views/session_summary_view.py | 107 +++++++++++++++ 4 files changed, 305 insertions(+) create mode 100644 app/views/connection_summary_view.py create mode 100644 app/views/session_summary_view.py diff --git a/app/urls.py b/app/urls.py index e95228c..5b32660 100644 --- a/app/urls.py +++ b/app/urls.py @@ -9,6 +9,8 @@ from .views.conference_duration_summary_view import ConferenceDurationSummaryView from .views.conference_participant_count_summary_view import ConferenceParticipantCountSummaryView from .views.issue_summary_view import IssueSummaryView, GetUserMediaSummaryView +from .views.connection_summary_view import ConnectionSummaryView, ConnectionSetupTimeSummaryView +from .views.session_summary_view import SessionSummaryView from .views.connection_event_view import ConnectionEventView from .views.connection_view import ConnectionView from .views.issue_view import IssueView @@ -40,6 +42,8 @@ path('connection/batch', ConnectionEventBatchView.as_view(), name='connection-batch'), path('connections', ConnectionView.as_view(), name='connections'), + path('connections/summary', ConnectionSummaryView.as_view(), name='connections-summary'), + path('connections/setup-time-summary', ConnectionSetupTimeSummaryView.as_view(), name='connections-setup-time-summary'), path('connections/', ConnectionView.as_view(), name='connection'), path('issues', IssueView.as_view(), name='issues'), path('issues/summary', IssueSummaryView.as_view(), name='issues-summary'), @@ -50,6 +54,7 @@ path('tracks', TracksView.as_view(), name='tracks'), path('sessions', SessionView.as_view(), name='sessions'), + path('sessions/summary', SessionSummaryView.as_view(), name='sessions-summary'), path('sessions/', SessionView.as_view(), name='session'), path('organizations', OrganizatonsView.as_view(), name='organizations'), diff --git a/app/views/conference_view.py b/app/views/conference_view.py index f08e7d2..f7fb3da 100644 --- a/app/views/conference_view.py +++ b/app/views/conference_view.py @@ -39,6 +39,12 @@ def filter(cls, request): if request.GET.get(rkey): filters[key] = request.GET.get(rkey) + ids_param = request.GET.get('conference_ids') + if ids_param: + ids = [i for i in ids_param.split(',') if i] + if ids: + filters['id__in'] = ids + if not filters: raise PMError(status=400, app_error=MISSING_PARAMETERS) diff --git a/app/views/connection_summary_view.py b/app/views/connection_summary_view.py new file mode 100644 index 0000000..26ef66b --- /dev/null +++ b/app/views/connection_summary_view.py @@ -0,0 +1,187 @@ +import datetime +from collections import Counter + +from django.core.exceptions import ValidationError +from django.db.models import Case, Count, IntegerField, When + +from ..errors import INVALID_PARAMETERS, MISSING_PARAMETERS, PMError +from ..utils import JSONHttpResponse +from ..models.connection import Connection, TYPE_OF_CONNECTIONS_ENUM +from .generic_view import GenericView + + +SETUP_TIME_BUCKETS = [ + {'title': '< 250 ms', 'min_ms': 0, 'max_ms': 250}, + {'title': '250 - 500 ms', 'min_ms': 250, 'max_ms': 500}, + {'title': '500 - 750 ms', 'min_ms': 500, 'max_ms': 750}, + {'title': '750 - 1000 ms', 'min_ms': 750, 'max_ms': 1000}, + {'title': '1000 - 1500 ms', 'min_ms': 1000, 'max_ms': 1500}, + {'title': '1500 - 2000 ms', 'min_ms': 1500, 'max_ms': 2000}, + {'title': '2000 - 2500 ms', 'min_ms': 2000, 'max_ms': 2500}, + {'title': '2500 - 3000 ms', 'min_ms': 2500, 'max_ms': 3000}, + {'title': '3000 - 4000 ms', 'min_ms': 3000, 'max_ms': 4000}, + {'title': '4000 - 5000 ms', 'min_ms': 4000, 'max_ms': 5000}, + {'title': '> 5000 ms', 'min_ms': 5000, 'max_ms': None}, +] + + +def _parse_iso(value): + if value.endswith('Z'): + value = value[:-1] + '+00:00' + return datetime.datetime.fromisoformat(value) + + +def _parse_time(value): + if not value: + return None + try: + return _parse_iso(value) if isinstance(value, str) else value + except Exception: + return None + + +def _bucket_for(ms): + for i, b in enumerate(SETUP_TIME_BUCKETS): + if b['max_ms'] is None: + if ms >= b['min_ms']: + return i + elif b['min_ms'] <= ms < b['max_ms']: + return i + return None + + +class ConnectionSummaryView(GenericView): + """ + Returns connection counts grouped by type — relay (TURN) vs direct. + Replaces the Relayed-connections pie chart's client-side aggregation. + """ + + @classmethod + def get(cls, request): + app_id = request.GET.get('appId') + if not app_id: + raise PMError(status=400, app_error=MISSING_PARAMETERS) + + filters = { + 'conference__app_id': app_id, + 'is_active': True, + } + + created_at_gte = request.GET.get('created_at_gte') + if created_at_gte: + try: + filters['created_at__gte'] = _parse_iso(created_at_gte) + except ValueError: + raise PMError(status=400, app_error=INVALID_PARAMETERS) + + created_at_lte = request.GET.get('created_at_lte') + if created_at_lte: + try: + filters['created_at__lte'] = _parse_iso(created_at_lte) + except ValueError: + raise PMError(status=400, app_error=INVALID_PARAMETERS) + + relay_code = TYPE_OF_CONNECTIONS_ENUM['relay'] + + try: + rows = (Connection.objects + .filter(**filters) + .exclude(type__isnull=True) + .annotate( + group=Case( + When(type=relay_code, then=1), + default=0, + output_field=IntegerField(), + ), + ) + .values('group') + .annotate(count=Count('id'))) + except ValidationError: + raise PMError(status=400, app_error=INVALID_PARAMETERS) + + counts = {0: 0, 1: 0} + for row in rows: + counts[row['group']] = row['count'] + + return JSONHttpResponse({ + 'data': [ + {'name': 'Direct', 'count': counts[0]}, + {'name': 'Relayed', 'count': counts[1]}, + ], + }) + + +class ConnectionSetupTimeSummaryView(GenericView): + """ + Returns counts of connections bucketed by initial-negotiation setup + time (end_time - start_time on the first 'connected' negotiation + in connection_info.negotiations). + """ + + @classmethod + def get(cls, request): + app_id = request.GET.get('appId') + if not app_id: + raise PMError(status=400, app_error=MISSING_PARAMETERS) + + filters = { + 'conference__app_id': app_id, + 'is_active': True, + } + + created_at_gte = request.GET.get('created_at_gte') + if created_at_gte: + try: + filters['created_at__gte'] = _parse_iso(created_at_gte) + except ValueError: + raise PMError(status=400, app_error=INVALID_PARAMETERS) + + created_at_lte = request.GET.get('created_at_lte') + if created_at_lte: + try: + filters['created_at__lte'] = _parse_iso(created_at_lte) + except ValueError: + raise PMError(status=400, app_error=INVALID_PARAMETERS) + + counters = Counter() + conferences_per_bucket = {i: set() for i in range(len(SETUP_TIME_BUCKETS))} + + try: + rows = Connection.objects.filter(**filters).values_list( + 'connection_info', 'conference_id' + ) + for info, conf_id in rows: + if not info: + continue + negotiations = info.get('negotiations') or [] + if not negotiations: + continue + first = negotiations[0] + if first.get('status') != 'connected': + continue + start = _parse_time(first.get('start_time')) + end = _parse_time(first.get('end_time')) + if not start or not end: + continue + ms = (end - start).total_seconds() * 1000 + if ms < 0: + continue + idx = _bucket_for(ms) + if idx is None: + continue + counters[idx] += 1 + conferences_per_bucket[idx].add(str(conf_id)) + except ValidationError: + raise PMError(status=400, app_error=INVALID_PARAMETERS) + + data = [] + for i, b in enumerate(SETUP_TIME_BUCKETS): + data.append({ + 'range': b['title'], + 'min_ms': b['min_ms'], + 'max_ms': b['max_ms'], + 'count': counters.get(i, 0), + 'conference_ids': list(conferences_per_bucket[i]), + }) + + return JSONHttpResponse({'data': data}) diff --git a/app/views/session_summary_view.py b/app/views/session_summary_view.py new file mode 100644 index 0000000..6b81eda --- /dev/null +++ b/app/views/session_summary_view.py @@ -0,0 +1,107 @@ +import datetime +from collections import Counter, defaultdict + +from django.core.exceptions import ValidationError + +from ..errors import INVALID_PARAMETERS, MISSING_PARAMETERS, PMError +from ..utils import JSONHttpResponse +from ..models.session import Session +from .generic_view import GenericView + + +def _parse_iso(value): + if value.endswith('Z'): + value = value[:-1] + '+00:00' + return datetime.datetime.fromisoformat(value) + + +class SessionSummaryView(GenericView): + """ + Returns session counts aggregated by browser, OS, and country. + Replaces downloading all sessions to aggregate client-side in the + Browsers, Operating Systems, and Map charts. + + Also returns geo points (lat/lon) for rendering on the map chart. + """ + + @classmethod + def get(cls, request): + app_id = request.GET.get('appId') + if not app_id: + raise PMError(status=400, app_error=MISSING_PARAMETERS) + + filters = { + 'conference__app_id': app_id, + 'is_active': True, + } + + created_at_gte = request.GET.get('created_at_gte') + if created_at_gte: + try: + filters['created_at__gte'] = _parse_iso(created_at_gte) + except ValueError: + raise PMError(status=400, app_error=INVALID_PARAMETERS) + + created_at_lte = request.GET.get('created_at_lte') + if created_at_lte: + try: + filters['created_at__lte'] = _parse_iso(created_at_lte) + except ValueError: + raise PMError(status=400, app_error=INVALID_PARAMETERS) + + browsers = Counter() + oses = Counter() + countries = Counter() + cities = {} # city name -> {lat, lon, count} + + try: + rows = Session.objects.filter(**filters).values_list('platform', 'geo_ip') + for platform, geo_ip in rows: + if platform: + browser = (platform.get('browser') or {}).get('name') or 'Unknown' + os_name = (platform.get('os') or {}).get('name') or 'Unknown' + browsers[browser] += 1 + oses[os_name] += 1 + + if geo_ip: + country_code = geo_ip.get('country_code') + if country_code: + countries[country_code] += 1 + + city = geo_ip.get('city') + lat = geo_ip.get('latitude') + lon = geo_ip.get('longitude') + if city and lat is not None and lon is not None: + try: + lat_f = float(lat) + lon_f = float(lon) + if lat_f or lon_f: + if city in cities: + cities[city]['count'] += 1 + else: + cities[city] = { + 'city': city, + 'latitude': lat_f, + 'longitude': lon_f, + 'count': 1, + } + except (TypeError, ValueError): + pass + except ValidationError: + raise PMError(status=400, app_error=INVALID_PARAMETERS) + + return JSONHttpResponse({ + 'browsers': [ + {'name': k, 'count': v} + for k, v in browsers.most_common() + ], + 'os': [ + {'name': k, 'count': v} + for k, v in oses.most_common() + ], + 'countries': [ + {'code': k, 'count': v} + for k, v in countries.most_common() + ], + 'cities': list(cities.values()), + }) From 6b2a3a663119109ffd16beff16122f1c93f6e99d Mon Sep 17 00:00:00 2001 From: Samson Nkrumah Date: Thu, 23 Apr 2026 15:09:47 +0100 Subject: [PATCH 3/9] feat: 60s Redis cache + pre-warm for dashboard summary endpoints (Phase C of #20) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With Phases 0-5 merged, every dashboard chart reads from a server-side aggregation endpoint. The SQL is fast with indexes, but the same ~8 queries run on every page load, and the heavy ones (sessions.summary, connections.setup_time_summary) still cost 400-800ms on a live tenant. Adds a thin caching layer in front of each summary view: - `app/summary_cache.py` — `cached_json(endpoint, request, compute)` hashes (endpoint + filter params) into a short key, reads Redis, falls through to `compute()` on miss, and writes back with a 60s TTL. Redis failures are tolerated (settings already has IGNORE_EXCEPTIONS). - Each of the eight summary views moves its existing compute body into a local `compute()` closure and returns through the helper. No change to the JSON shape, query logic, or error handling. - `manage.py prewarm_summaries` — scheduled command that iterates apps with recent traffic (default: any conference in the last 2 days) and runs every summary view with the 30d-window filters the dashboard sends by default. Intended to run every ~30s as an ECS scheduled task so first visitors never see a cold miss. Measured locally against a 7-day Production clone (~18k conferences / 38k sessions / 38k connections): endpoint cold warm conferences/summary 391ms → 12ms (33x) sessions/summary 748ms → 11ms (68x) connections/setup_time_summary 373ms → 11ms (34x) conferences/participant_count_summary 216ms → 7ms (31x) issues/gum_summary 107ms → 6ms (18x) connections/summary 57ms → 6ms (9.5x) issues/summary 45ms → 86ms (noise; both <100ms) conferences/duration_summary 19ms → 8ms (2.3x) Co-Authored-By: Claude Opus 4.7 (1M context) --- app/management/commands/prewarm_summaries.py | 98 ++++++++++++ app/summary_cache.py | 75 +++++++++ app/views/conference_duration_summary_view.py | 67 ++++---- ...nference_participant_count_summary_view.py | 60 ++++---- app/views/conference_summary_view.py | 87 ++++++----- app/views/connection_summary_view.py | 143 +++++++++--------- app/views/issue_summary_view.py | 92 ++++++----- app/views/session_summary_view.py | 117 +++++++------- 8 files changed, 472 insertions(+), 267 deletions(-) create mode 100644 app/management/commands/prewarm_summaries.py create mode 100644 app/summary_cache.py diff --git a/app/management/commands/prewarm_summaries.py b/app/management/commands/prewarm_summaries.py new file mode 100644 index 0000000..b633fcc --- /dev/null +++ b/app/management/commands/prewarm_summaries.py @@ -0,0 +1,98 @@ +""" +Pre-warm the dashboard-summary Redis cache so first visitors don't pay +the cold-query tax. Intended to run every ~30s via ECS scheduled task. + +For each active app that has seen recent data, walks every summary view +with the same (appId, created_at_gte) filter the dashboard sends by +default (last 30 days). The views themselves populate the cache on miss. +""" +import datetime +import logging +import time + +from django.core.management.base import BaseCommand +from django.test.client import RequestFactory + +from app.models.app import App +from app.models.conference import Conference + +from app.views.conference_summary_view import ConferenceSummaryView +from app.views.conference_duration_summary_view import ConferenceDurationSummaryView +from app.views.conference_participant_count_summary_view import ConferenceParticipantCountSummaryView +from app.views.issue_summary_view import IssueSummaryView, GetUserMediaSummaryView +from app.views.connection_summary_view import ConnectionSummaryView, ConnectionSetupTimeSummaryView +from app.views.session_summary_view import SessionSummaryView + +logger = logging.getLogger(__name__) + +# (label, view) +VIEWS = [ + ('conferences.summary', ConferenceSummaryView), + ('conferences.duration_summary', ConferenceDurationSummaryView), + ('conferences.participant_count_summary', ConferenceParticipantCountSummaryView), + ('issues.summary', IssueSummaryView), + ('issues.gum_summary', GetUserMediaSummaryView), + ('connections.summary', ConnectionSummaryView), + ('connections.setup_time_summary', ConnectionSetupTimeSummaryView), + ('sessions.summary', SessionSummaryView), +] + + +class Command(BaseCommand): + help = 'Pre-compute dashboard summary responses and cache them' + + def add_arguments(self, parser): + parser.add_argument( + '--days', + type=int, + default=30, + help='Window to warm (default: 30 days, matches dashboard default)', + ) + parser.add_argument( + '--active-within-days', + type=int, + default=2, + help='Only warm apps that saw a conference in the last N days (default: 2)', + ) + + def handle(self, *args, **options): + window_days = options['days'] + active_within = options['active_within_days'] + + since_window = datetime.datetime.utcnow() - datetime.timedelta(days=window_days) + active_since = datetime.datetime.utcnow() - datetime.timedelta(days=active_within) + + # Apps with any conference in the recent window — skip tenants with + # no traffic so warming doesn't scan their cold tables. + recent_app_ids = (Conference.objects + .filter(created_at__gte=active_since) + .values_list('app_id', flat=True) + .distinct()) + apps = App.objects.filter(id__in=list(recent_app_ids), is_active=True) + count = apps.count() + self.stdout.write(f'Warming {count} active apps ({window_days}d window)') + + rf = RequestFactory() + created_at_gte = since_window.isoformat() + 'Z' + warmed = 0 + failed = 0 + + for app in apps: + for label, view_cls in VIEWS: + req = rf.get('/v1/' + label, { + 'appId': str(app.id), + 'created_at_gte': created_at_gte, + }) + started = time.monotonic() + try: + view_cls.get(req) + warmed += 1 + except Exception as e: + failed += 1 + logger.warning('prewarm %s for app %s failed: %s', label, app.id, e) + elapsed_ms = (time.monotonic() - started) * 1000 + if elapsed_ms > 500: + self.stdout.write(f' slow: {label} app={app.id} {elapsed_ms:.0f}ms') + + self.stdout.write(f'Warmed {warmed} summary entries across {count} apps' + + (f' ({failed} failed)' if failed else '')) diff --git a/app/summary_cache.py b/app/summary_cache.py new file mode 100644 index 0000000..33940ff --- /dev/null +++ b/app/summary_cache.py @@ -0,0 +1,75 @@ +""" +Short-TTL Redis cache for dashboard summary endpoints. + +The summary endpoints run SQL aggregations that are fast enough on their own +but get called by every dashboard page load. Cache the computed JSON for +~60 seconds so concurrent viewers share the same roll-up. + +Keys are derived from the endpoint name + all request filters, so different +date ranges / apps get independent entries. TTL-only — no explicit +invalidation — because the data is strictly additive (new conferences, +sessions, issues arrive over time) and a sub-minute staleness window is +acceptable for a dashboard. +""" +import hashlib +import json +import logging + +from django.conf import settings +from django.core.cache import cache + +logger = logging.getLogger(__name__) + +DEFAULT_TTL_SECONDS = 60 +KEY_PREFIX = 'summary' + +# The query-string params that factor into the cache key for each endpoint. +# Anything not listed here is ignored (e.g. trailing slashes, user agent, etc). +CACHE_KEY_PARAMS = ( + 'appId', + 'created_at_gte', + 'created_at_lte', + 'conferenceId', + 'participantId', +) + + +def _make_key(endpoint, request): + parts = [endpoint] + for name in CACHE_KEY_PARAMS: + val = request.GET.get(name) + if val: + parts.append(f'{name}={val}') + raw = '|'.join(parts) + # Keep key short but unique; include a readable prefix for ops visibility. + digest = hashlib.sha1(raw.encode()).hexdigest()[:16] + return f'{KEY_PREFIX}:{endpoint}:{digest}' + + +def get_ttl(): + return getattr(settings, 'SUMMARY_CACHE_TTL', DEFAULT_TTL_SECONDS) + + +def cached_json(endpoint, request, compute): + """ + Returns (payload_dict, was_cached_bool). + + `compute` is called only on cache miss and must return the JSON-serializable + dict the endpoint would have returned. + """ + key = _make_key(endpoint, request) + try: + cached = cache.get(key) + except Exception as e: + logger.warning('summary cache get failed for %s: %s', key, e) + cached = None + + if cached is not None: + return cached, True + + payload = compute() + try: + cache.set(key, payload, timeout=get_ttl()) + except Exception as e: + logger.warning('summary cache set failed for %s: %s', key, e) + return payload, False diff --git a/app/views/conference_duration_summary_view.py b/app/views/conference_duration_summary_view.py index b12e312..e62a9e8 100644 --- a/app/views/conference_duration_summary_view.py +++ b/app/views/conference_duration_summary_view.py @@ -4,6 +4,7 @@ from django.db.models import Case, Count, IntegerField, When from ..errors import INVALID_PARAMETERS, MISSING_PARAMETERS, PMError +from ..summary_cache import cached_json from ..utils import JSONHttpResponse from ..models.conference import Conference from .generic_view import GenericView @@ -56,35 +57,37 @@ def get(cls, request): except ValueError: raise PMError(status=400, app_error=INVALID_PARAMETERS) - whens = [] - for i, b in enumerate(BUCKETS): - lo, hi = b['min_sec'], b['max_sec'] - if hi is None: - whens.append(When(duration__gte=lo, then=i)) - elif lo == 0: - whens.append(When(duration__lt=hi, then=i)) - else: - whens.append(When(duration__gte=lo, duration__lt=hi, then=i)) - - try: - rows = (Conference.objects - .filter(**filters) - .annotate(bucket=Case(*whens, output_field=IntegerField())) - .values('bucket') - .annotate(count=Count('id')) - .order_by('bucket')) - except ValidationError: - raise PMError(status=400, app_error=INVALID_PARAMETERS) - - counts_by_bucket = {r['bucket']: r['count'] for r in rows if r['bucket'] is not None} - data = [ - { - 'range': b['title'], - 'min_sec': b['min_sec'], - 'max_sec': b['max_sec'], - 'count': counts_by_bucket.get(i, 0), - } - for i, b in enumerate(BUCKETS) - ] - - return JSONHttpResponse({'data': data}) + def compute(): + whens = [] + for i, b in enumerate(BUCKETS): + lo, hi = b['min_sec'], b['max_sec'] + if hi is None: + whens.append(When(duration__gte=lo, then=i)) + elif lo == 0: + whens.append(When(duration__lt=hi, then=i)) + else: + whens.append(When(duration__gte=lo, duration__lt=hi, then=i)) + + try: + rows = (Conference.objects + .filter(**filters) + .annotate(bucket=Case(*whens, output_field=IntegerField())) + .values('bucket') + .annotate(count=Count('id')) + .order_by('bucket')) + except ValidationError: + raise PMError(status=400, app_error=INVALID_PARAMETERS) + + counts_by_bucket = {r['bucket']: r['count'] for r in rows if r['bucket'] is not None} + return {'data': [ + { + 'range': b['title'], + 'min_sec': b['min_sec'], + 'max_sec': b['max_sec'], + 'count': counts_by_bucket.get(i, 0), + } + for i, b in enumerate(BUCKETS) + ]} + + payload, _ = cached_json('conferences.duration_summary', request, compute) + return JSONHttpResponse(payload) diff --git a/app/views/conference_participant_count_summary_view.py b/app/views/conference_participant_count_summary_view.py index a5bf7ec..188d916 100644 --- a/app/views/conference_participant_count_summary_view.py +++ b/app/views/conference_participant_count_summary_view.py @@ -5,6 +5,7 @@ from django.db.models import Count, IntegerField, OuterRef, Subquery from ..errors import INVALID_PARAMETERS, MISSING_PARAMETERS, PMError +from ..summary_cache import cached_json from ..utils import JSONHttpResponse from ..models.conference import Conference from ..models.participant import Participant @@ -45,33 +46,38 @@ def get(cls, request): except ValueError: raise PMError(status=400, app_error=INVALID_PARAMETERS) - try: - per_conf = (Conference.objects - .filter(**filters) - .annotate( - participants_count=Subquery( - Participant.objects.filter( - conferences=OuterRef('pk'), - is_active=True, - ).order_by().values('conferences').annotate( - cnt=Count('id', distinct=True) - ).values('cnt')[:1], - output_field=IntegerField(), - ), - ) - .values_list('participants_count', flat=True)) - except ValidationError: - raise PMError(status=400, app_error=INVALID_PARAMETERS) + def compute(): + try: + per_conf = (Conference.objects + .filter(**filters) + .annotate( + participants_count=Subquery( + Participant.objects.filter( + conferences=OuterRef('pk'), + is_active=True, + ).order_by().values('conferences').annotate( + cnt=Count('id', distinct=True) + ).values('cnt')[:1], + output_field=IntegerField(), + ), + ) + .values_list('participants_count', flat=True)) + except ValidationError: + raise PMError(status=400, app_error=INVALID_PARAMETERS) - distribution = Counter() - total = 0 - for count in per_conf: - distribution[count or 0] += 1 - total += 1 + distribution = Counter() + total = 0 + for count in per_conf: + distribution[count or 0] += 1 + total += 1 - data = [ - {'participants': n, 'conferences': c} - for n, c in sorted(distribution.items()) - ] + return { + 'data': [ + {'participants': n, 'conferences': c} + for n, c in sorted(distribution.items()) + ], + 'total_conferences': total, + } - return JSONHttpResponse({'data': data, 'total_conferences': total}) + payload, _ = cached_json('conferences.participant_count_summary', request, compute) + return JSONHttpResponse(payload) diff --git a/app/views/conference_summary_view.py b/app/views/conference_summary_view.py index ec90008..d37b201 100644 --- a/app/views/conference_summary_view.py +++ b/app/views/conference_summary_view.py @@ -6,6 +6,7 @@ from django.db.models.functions import TruncDate from ..errors import INVALID_PARAMETERS, MISSING_PARAMETERS, PMError +from ..summary_cache import cached_json from ..utils import JSONHttpResponse from ..models.conference import Conference from ..models.issue import Issue @@ -48,48 +49,50 @@ def get(cls, request): except ValueError: raise PMError(status=400, app_error=INVALID_PARAMETERS) - try: - rows = (Conference.objects - .filter(**filters) - .annotate( - day=TruncDate('created_at'), - has_error=Exists( - Issue.objects.filter(conference=OuterRef('pk'), type='e', is_active=True) - ), - has_warning=Exists( - Issue.objects.filter(conference=OuterRef('pk'), type='w', is_active=True) - ), - ) - .annotate( - status=Case( - When(ongoing=True, then=Value('ongoing')), - When(has_error=True, then=Value('error')), - When(has_warning=True, then=Value('warning')), - default=Value('success'), - output_field=CharField(), - ), - ) - .values('day', 'status') - .annotate(count=Count('id')) - .order_by('day')) - except ValidationError: - raise PMError(status=400, app_error=INVALID_PARAMETERS) + def compute(): + try: + rows = (Conference.objects + .filter(**filters) + .annotate( + day=TruncDate('created_at'), + has_error=Exists( + Issue.objects.filter(conference=OuterRef('pk'), type='e', is_active=True) + ), + has_warning=Exists( + Issue.objects.filter(conference=OuterRef('pk'), type='w', is_active=True) + ), + ) + .annotate( + status=Case( + When(ongoing=True, then=Value('ongoing')), + When(has_error=True, then=Value('error')), + When(has_warning=True, then=Value('warning')), + default=Value('success'), + output_field=CharField(), + ), + ) + .values('day', 'status') + .annotate(count=Count('id')) + .order_by('day')) + except ValidationError: + raise PMError(status=400, app_error=INVALID_PARAMETERS) - buckets = defaultdict(lambda: {'success': 0, 'warning': 0, 'error': 0, 'ongoing': 0}) - for row in rows: - day_key = row['day'].isoformat() if row['day'] else None - buckets[day_key][row['status']] = row['count'] + buckets = defaultdict(lambda: {'success': 0, 'warning': 0, 'error': 0, 'ongoing': 0}) + for row in rows: + day_key = row['day'].isoformat() if row['day'] else None + buckets[day_key][row['status']] = row['count'] - data = [ - { - 'date': day, - 'success': counts['success'], - 'warning': counts['warning'], - 'error': counts['error'], - 'ongoing': counts['ongoing'], - 'total': counts['success'] + counts['warning'] + counts['error'] + counts['ongoing'], - } - for day, counts in sorted(buckets.items(), key=lambda x: x[0] or '') - ] + return {'data': [ + { + 'date': day, + 'success': counts['success'], + 'warning': counts['warning'], + 'error': counts['error'], + 'ongoing': counts['ongoing'], + 'total': counts['success'] + counts['warning'] + counts['error'] + counts['ongoing'], + } + for day, counts in sorted(buckets.items(), key=lambda x: x[0] or '') + ]} - return JSONHttpResponse({'data': data}) + payload, _ = cached_json('conferences.summary', request, compute) + return JSONHttpResponse(payload) diff --git a/app/views/connection_summary_view.py b/app/views/connection_summary_view.py index 26ef66b..4969642 100644 --- a/app/views/connection_summary_view.py +++ b/app/views/connection_summary_view.py @@ -5,6 +5,7 @@ from django.db.models import Case, Count, IntegerField, When from ..errors import INVALID_PARAMETERS, MISSING_PARAMETERS, PMError +from ..summary_cache import cached_json from ..utils import JSONHttpResponse from ..models.connection import Connection, TYPE_OF_CONNECTIONS_ENUM from .generic_view import GenericView @@ -81,34 +82,37 @@ def get(cls, request): except ValueError: raise PMError(status=400, app_error=INVALID_PARAMETERS) - relay_code = TYPE_OF_CONNECTIONS_ENUM['relay'] - - try: - rows = (Connection.objects - .filter(**filters) - .exclude(type__isnull=True) - .annotate( - group=Case( - When(type=relay_code, then=1), - default=0, - output_field=IntegerField(), - ), - ) - .values('group') - .annotate(count=Count('id'))) - except ValidationError: - raise PMError(status=400, app_error=INVALID_PARAMETERS) + def compute(): + relay_code = TYPE_OF_CONNECTIONS_ENUM['relay'] + try: + rows = (Connection.objects + .filter(**filters) + .exclude(type__isnull=True) + .annotate( + group=Case( + When(type=relay_code, then=1), + default=0, + output_field=IntegerField(), + ), + ) + .values('group') + .annotate(count=Count('id'))) + except ValidationError: + raise PMError(status=400, app_error=INVALID_PARAMETERS) - counts = {0: 0, 1: 0} - for row in rows: - counts[row['group']] = row['count'] + counts = {0: 0, 1: 0} + for row in rows: + counts[row['group']] = row['count'] - return JSONHttpResponse({ - 'data': [ - {'name': 'Direct', 'count': counts[0]}, - {'name': 'Relayed', 'count': counts[1]}, - ], - }) + return { + 'data': [ + {'name': 'Direct', 'count': counts[0]}, + {'name': 'Relayed', 'count': counts[1]}, + ], + } + + payload, _ = cached_json('connections.summary', request, compute) + return JSONHttpResponse(payload) class ConnectionSetupTimeSummaryView(GenericView): @@ -143,45 +147,48 @@ def get(cls, request): except ValueError: raise PMError(status=400, app_error=INVALID_PARAMETERS) - counters = Counter() - conferences_per_bucket = {i: set() for i in range(len(SETUP_TIME_BUCKETS))} - - try: - rows = Connection.objects.filter(**filters).values_list( - 'connection_info', 'conference_id' - ) - for info, conf_id in rows: - if not info: - continue - negotiations = info.get('negotiations') or [] - if not negotiations: - continue - first = negotiations[0] - if first.get('status') != 'connected': - continue - start = _parse_time(first.get('start_time')) - end = _parse_time(first.get('end_time')) - if not start or not end: - continue - ms = (end - start).total_seconds() * 1000 - if ms < 0: - continue - idx = _bucket_for(ms) - if idx is None: - continue - counters[idx] += 1 - conferences_per_bucket[idx].add(str(conf_id)) - except ValidationError: - raise PMError(status=400, app_error=INVALID_PARAMETERS) - - data = [] - for i, b in enumerate(SETUP_TIME_BUCKETS): - data.append({ - 'range': b['title'], - 'min_ms': b['min_ms'], - 'max_ms': b['max_ms'], - 'count': counters.get(i, 0), - 'conference_ids': list(conferences_per_bucket[i]), - }) - - return JSONHttpResponse({'data': data}) + def compute(): + counters = Counter() + conferences_per_bucket = {i: set() for i in range(len(SETUP_TIME_BUCKETS))} + + try: + rows = Connection.objects.filter(**filters).values_list( + 'connection_info', 'conference_id' + ) + for info, conf_id in rows: + if not info: + continue + negotiations = info.get('negotiations') or [] + if not negotiations: + continue + first = negotiations[0] + if first.get('status') != 'connected': + continue + start = _parse_time(first.get('start_time')) + end = _parse_time(first.get('end_time')) + if not start or not end: + continue + ms = (end - start).total_seconds() * 1000 + if ms < 0: + continue + idx = _bucket_for(ms) + if idx is None: + continue + counters[idx] += 1 + conferences_per_bucket[idx].add(str(conf_id)) + except ValidationError: + raise PMError(status=400, app_error=INVALID_PARAMETERS) + + data = [] + for i, b in enumerate(SETUP_TIME_BUCKETS): + data.append({ + 'range': b['title'], + 'min_ms': b['min_ms'], + 'max_ms': b['max_ms'], + 'count': counters.get(i, 0), + 'conference_ids': list(conferences_per_bucket[i]), + }) + return {'data': data} + + payload, _ = cached_json('connections.setup_time_summary', request, compute) + return JSONHttpResponse(payload) diff --git a/app/views/issue_summary_view.py b/app/views/issue_summary_view.py index b850e59..cc144fb 100644 --- a/app/views/issue_summary_view.py +++ b/app/views/issue_summary_view.py @@ -4,6 +4,7 @@ from django.db.models import Count from ..errors import INVALID_PARAMETERS, MISSING_PARAMETERS, PMError +from ..summary_cache import cached_json from ..utils import JSONHttpResponse from ..models.issue import Issue, ISSUES from .generic_view import GenericView @@ -47,25 +48,27 @@ def get(cls, request): except ValueError: raise PMError(status=400, app_error=INVALID_PARAMETERS) - try: - rows = (Issue.objects - .filter(**filters) - .values('code') - .annotate(count=Count('id')) - .order_by('-count')) - except ValidationError: - raise PMError(status=400, app_error=INVALID_PARAMETERS) - - data = [ - { - 'code': r['code'], - 'title': ISSUES.get(r['code'], {}).get('title', r['code']), - 'count': r['count'], - } - for r in rows - ] + def compute(): + try: + rows = (Issue.objects + .filter(**filters) + .values('code') + .annotate(count=Count('id')) + .order_by('-count')) + except ValidationError: + raise PMError(status=400, app_error=INVALID_PARAMETERS) + + return {'data': [ + { + 'code': r['code'], + 'title': ISSUES.get(r['code'], {}).get('title', r['code']), + 'count': r['count'], + } + for r in rows + ]} - return JSONHttpResponse({'data': data}) + payload, _ = cached_json('issues.summary', request, compute) + return JSONHttpResponse(payload) class GetUserMediaSummaryView(GenericView): @@ -100,29 +103,34 @@ def get(cls, request): except ValueError: raise PMError(status=400, app_error=INVALID_PARAMETERS) - from collections import Counter - name_counter = Counter() - message_by_name = {} - - try: - issues = Issue.objects.filter(**filters).values_list('data', flat=True) - for data in issues: - if not data: - continue - name = data.get('name') or 'Unknown' - name_counter[name] += 1 - if name not in message_by_name and data.get('message'): - message_by_name[name] = data['message'] - except ValidationError: - raise PMError(status=400, app_error=INVALID_PARAMETERS) - - data = [ - { - 'name': name, - 'message': message_by_name.get(name, ''), - 'count': count, + def compute(): + from collections import Counter + name_counter = Counter() + message_by_name = {} + + try: + issues = Issue.objects.filter(**filters).values_list('data', flat=True) + for data in issues: + if not data: + continue + name = data.get('name') or 'Unknown' + name_counter[name] += 1 + if name not in message_by_name and data.get('message'): + message_by_name[name] = data['message'] + except ValidationError: + raise PMError(status=400, app_error=INVALID_PARAMETERS) + + return { + 'data': [ + { + 'name': name, + 'message': message_by_name.get(name, ''), + 'count': count, + } + for name, count in name_counter.most_common() + ], + 'total': sum(name_counter.values()), } - for name, count in name_counter.most_common() - ] - return JSONHttpResponse({'data': data, 'total': sum(name_counter.values())}) + payload, _ = cached_json('issues.gum_summary', request, compute) + return JSONHttpResponse(payload) diff --git a/app/views/session_summary_view.py b/app/views/session_summary_view.py index 6b81eda..28f2406 100644 --- a/app/views/session_summary_view.py +++ b/app/views/session_summary_view.py @@ -4,6 +4,7 @@ from django.core.exceptions import ValidationError from ..errors import INVALID_PARAMETERS, MISSING_PARAMETERS, PMError +from ..summary_cache import cached_json from ..utils import JSONHttpResponse from ..models.session import Session from .generic_view import GenericView @@ -49,59 +50,63 @@ def get(cls, request): except ValueError: raise PMError(status=400, app_error=INVALID_PARAMETERS) - browsers = Counter() - oses = Counter() - countries = Counter() - cities = {} # city name -> {lat, lon, count} - - try: - rows = Session.objects.filter(**filters).values_list('platform', 'geo_ip') - for platform, geo_ip in rows: - if platform: - browser = (platform.get('browser') or {}).get('name') or 'Unknown' - os_name = (platform.get('os') or {}).get('name') or 'Unknown' - browsers[browser] += 1 - oses[os_name] += 1 - - if geo_ip: - country_code = geo_ip.get('country_code') - if country_code: - countries[country_code] += 1 - - city = geo_ip.get('city') - lat = geo_ip.get('latitude') - lon = geo_ip.get('longitude') - if city and lat is not None and lon is not None: - try: - lat_f = float(lat) - lon_f = float(lon) - if lat_f or lon_f: - if city in cities: - cities[city]['count'] += 1 - else: - cities[city] = { - 'city': city, - 'latitude': lat_f, - 'longitude': lon_f, - 'count': 1, - } - except (TypeError, ValueError): - pass - except ValidationError: - raise PMError(status=400, app_error=INVALID_PARAMETERS) - - return JSONHttpResponse({ - 'browsers': [ - {'name': k, 'count': v} - for k, v in browsers.most_common() - ], - 'os': [ - {'name': k, 'count': v} - for k, v in oses.most_common() - ], - 'countries': [ - {'code': k, 'count': v} - for k, v in countries.most_common() - ], - 'cities': list(cities.values()), - }) + def compute(): + browsers = Counter() + oses = Counter() + countries = Counter() + cities = {} # city name -> {lat, lon, count} + + try: + rows = Session.objects.filter(**filters).values_list('platform', 'geo_ip') + for platform, geo_ip in rows: + if platform: + browser = (platform.get('browser') or {}).get('name') or 'Unknown' + os_name = (platform.get('os') or {}).get('name') or 'Unknown' + browsers[browser] += 1 + oses[os_name] += 1 + + if geo_ip: + country_code = geo_ip.get('country_code') + if country_code: + countries[country_code] += 1 + + city = geo_ip.get('city') + lat = geo_ip.get('latitude') + lon = geo_ip.get('longitude') + if city and lat is not None and lon is not None: + try: + lat_f = float(lat) + lon_f = float(lon) + if lat_f or lon_f: + if city in cities: + cities[city]['count'] += 1 + else: + cities[city] = { + 'city': city, + 'latitude': lat_f, + 'longitude': lon_f, + 'count': 1, + } + except (TypeError, ValueError): + pass + except ValidationError: + raise PMError(status=400, app_error=INVALID_PARAMETERS) + + return { + 'browsers': [ + {'name': k, 'count': v} + for k, v in browsers.most_common() + ], + 'os': [ + {'name': k, 'count': v} + for k, v in oses.most_common() + ], + 'countries': [ + {'code': k, 'count': v} + for k, v in countries.most_common() + ], + 'cities': list(cities.values()), + } + + payload, _ = cached_json('sessions.summary', request, compute) + return JSONHttpResponse(payload) From 3f8694ff9a91629fc5623fec5feea3f93e9ca1c6 Mon Sep 17 00:00:00 2001 From: agonza1 Date: Fri, 24 Apr 2026 19:23:57 -0400 Subject: [PATCH 4/9] fix: dedupe conferences for issue_code; harden gum-summary --- README.md | 2 + app/tests/__init__.py | 0 app/tests/test_pr26_regressions.py | 143 +++++++++++++++++++++++++++++ app/views/conference_view.py | 6 ++ app/views/issue_summary_view.py | 2 +- 5 files changed, 152 insertions(+), 1 deletion(-) create mode 100644 app/tests/__init__.py create mode 100644 app/tests/test_pr26_regressions.py diff --git a/README.md b/README.md index d0a4181..fd911bb 100644 --- a/README.md +++ b/README.md @@ -987,6 +987,8 @@ Private endpoints are used by the web interface to query data. They require user - `GET`, query parameters: - `appId`: Filter by app - `participantId`: Filter by participant + - `issue_code`: Filter conferences that contain at least one active issue with this code + - Returns each conference once even if multiple matching issues exist - `/conferences/`: Get a specific conference - `GET` diff --git a/app/tests/__init__.py b/app/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/tests/test_pr26_regressions.py b/app/tests/test_pr26_regressions.py new file mode 100644 index 0000000..7e47b65 --- /dev/null +++ b/app/tests/test_pr26_regressions.py @@ -0,0 +1,143 @@ +import json + +from django.test import Client, TestCase + +from app.models.app import App +from app.models.conference import Conference +from app.models.issue import Issue +from app.models.organization import Organization +from app.models.participant import Participant +from app.models.session import Session + + +class PR26RegressionTests(TestCase): + def setUp(self): + self.client = Client() + self.organization = Organization.objects.create(name="Test Org") + self.app = App.objects.create( + name="Test App", + api_key="a" * 32, + organization=self.organization, + ) + + def _make_conference_graph(self, conference_id): + conference = Conference.objects.create( + conference_id=conference_id, + app=self.app, + ) + participant = Participant.objects.create( + participant_id=f"{conference_id}-participant", + app=self.app, + ) + participant.conferences.add(conference) + session = Session.objects.create( + conference=conference, + participant=participant, + ) + return conference, participant, session + + def test_conferences_issue_code_filter_returns_distinct_conferences(self): + conference, participant, session = self._make_conference_graph("conf-1") + + Issue.objects.create( + session=session, + conference=conference, + participant=participant, + type=Issue.TYPES_OF_ISSUES["warning"], + code="getusermedia_error", + data={"name": "NotFoundError"}, + ) + Issue.objects.create( + session=session, + conference=conference, + participant=participant, + type=Issue.TYPES_OF_ISSUES["warning"], + code="getusermedia_error", + data={"name": "NotFoundError"}, + ) + + response = self.client.get( + "/v1/conferences", + { + "appId": str(self.app.id), + "issue_code": "getusermedia_error", + "limit": "50", + }, + ) + + self.assertEqual(response.status_code, 200) + payload = json.loads(response.content) + self.assertEqual(payload["count"], 1) + self.assertEqual(len(payload["results"]), 1) + self.assertEqual(payload["results"][0]["id"], str(conference.id)) + + def test_conferences_issue_code_filter_ignores_inactive_issues(self): + conference_active, participant_active, session_active = self._make_conference_graph("conf-active") + conference_inactive, participant_inactive, session_inactive = self._make_conference_graph("conf-inactive") + + Issue.objects.create( + session=session_active, + conference=conference_active, + participant=participant_active, + type=Issue.TYPES_OF_ISSUES["warning"], + code="getusermedia_error", + data={"name": "NotFoundError"}, + is_active=True, + ) + Issue.objects.create( + session=session_inactive, + conference=conference_inactive, + participant=participant_inactive, + type=Issue.TYPES_OF_ISSUES["warning"], + code="getusermedia_error", + data={"name": "NotReadableError"}, + is_active=False, + ) + + response = self.client.get( + "/v1/conferences", + { + "appId": str(self.app.id), + "issue_code": "getusermedia_error", + "limit": "50", + }, + ) + + self.assertEqual(response.status_code, 200) + payload = json.loads(response.content) + returned_ids = {row["id"] for row in payload["results"]} + self.assertEqual(returned_ids, {str(conference_active.id)}) + self.assertEqual(payload["count"], 1) + + def test_gum_summary_skips_non_dict_issue_data(self): + conference, participant, session = self._make_conference_graph("conf-gum") + + Issue.objects.create( + session=session, + conference=conference, + participant=participant, + type=Issue.TYPES_OF_ISSUES["warning"], + code="getusermedia_error", + data={"name": "NotAllowedError", "message": "Permission denied"}, + ) + Issue.objects.create( + session=session, + conference=conference, + participant=participant, + type=Issue.TYPES_OF_ISSUES["warning"], + code="getusermedia_error", + data="malformed", + ) + + response = self.client.get( + "/v1/issues/gum-summary", + { + "appId": str(self.app.id), + }, + ) + + self.assertEqual(response.status_code, 200) + payload = json.loads(response.content) + self.assertEqual(payload["total"], 1) + self.assertEqual(payload["data"][0]["name"], "NotAllowedError") + self.assertEqual(payload["data"][0]["count"], 1) diff --git a/app/views/conference_view.py b/app/views/conference_view.py index f7fb3da..722899e 100644 --- a/app/views/conference_view.py +++ b/app/views/conference_view.py @@ -39,6 +39,10 @@ def filter(cls, request): if request.GET.get(rkey): filters[key] = request.GET.get(rkey) + filter_by_issue_code = bool(request.GET.get('issue_code')) + if filter_by_issue_code: + filters['issues__is_active'] = True + ids_param = request.GET.get('conference_ids') if ids_param: ids = [i for i in ids_param.split(',') if i] @@ -69,6 +73,8 @@ def filter(cls, request): output_field=IntegerField(), ), ) + if filter_by_issue_code: + objs = objs.distinct() except ValidationError: raise PMError(status=400, app_error=INVALID_PARAMETERS) diff --git a/app/views/issue_summary_view.py b/app/views/issue_summary_view.py index cc144fb..25c895b 100644 --- a/app/views/issue_summary_view.py +++ b/app/views/issue_summary_view.py @@ -111,7 +111,7 @@ def compute(): try: issues = Issue.objects.filter(**filters).values_list('data', flat=True) for data in issues: - if not data: + if not data or not isinstance(data, dict): continue name = data.get('name') or 'Unknown' name_counter[name] += 1 From e7e51f7766b5acc535546c09ee2a40c9a939c9d3 Mon Sep 17 00:00:00 2001 From: agonza1 Date: Fri, 24 Apr 2026 19:37:59 -0400 Subject: [PATCH 5/9] test: summary_cache LocMem tests and prewarm smoke - Unit-test cache key rules, hit/miss, TTL override, and soft-fail on get/set errors. - Smoke-test prewarm_summaries for zero apps and one recent app (8 views). Made-with: Cursor --- app/tests/test_prewarm_summaries.py | 41 +++++++++ app/tests/test_summary_cache.py | 135 ++++++++++++++++++++++++++++ 2 files changed, 176 insertions(+) create mode 100644 app/tests/test_prewarm_summaries.py create mode 100644 app/tests/test_summary_cache.py diff --git a/app/tests/test_prewarm_summaries.py b/app/tests/test_prewarm_summaries.py new file mode 100644 index 0000000..85492ed --- /dev/null +++ b/app/tests/test_prewarm_summaries.py @@ -0,0 +1,41 @@ +""" +Smoke tests for prewarm_summaries management command (PR #28). +""" +from io import StringIO + +from django.core.management import call_command +from django.test import TestCase + +from app.models.app import App +from app.models.conference import Conference +from app.models.organization import Organization + + +class PrewarmSummariesSmokeTests(TestCase): + def test_runs_with_no_qualifying_apps(self): + out = StringIO() + err = StringIO() + call_command('prewarm_summaries', stdout=out, stderr=err) + + combined = out.getvalue() + err.getvalue() + self.assertIn('Warming 0 active apps', combined) + self.assertIn('Warmed 0 summary entries across 0 apps', combined) + + def test_runs_for_one_app_with_recent_conference(self): + org = Organization.objects.create(name='Prewarm Org') + app = App.objects.create( + name='Prewarm App', + api_key='b' * 32, + organization=org, + ) + Conference.objects.create( + conference_id='prewarm-conf-1', + app=app, + ) + + out = StringIO() + call_command('prewarm_summaries', stdout=out, stderr=StringIO()) + text = out.getvalue() + + self.assertIn('Warming 1 active apps', text) + self.assertIn('Warmed 8 summary entries across 1 apps', text) diff --git a/app/tests/test_summary_cache.py b/app/tests/test_summary_cache.py new file mode 100644 index 0000000..cb35100 --- /dev/null +++ b/app/tests/test_summary_cache.py @@ -0,0 +1,135 @@ +""" +Unit tests for app.summary_cache (PR #28): LocMem backend, no Redis required. +""" +from unittest.mock import patch + +from django.core.cache import cache +from django.test import RequestFactory, SimpleTestCase, override_settings + +from app.summary_cache import _make_key, cached_json, get_ttl + + +@override_settings( + CACHES={ + 'default': { + 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', + 'LOCATION': 'test-summary-cache-locmem', + } + } +) +class SummaryCacheTests(SimpleTestCase): + def setUp(self): + cache.clear() + + def test_make_key_same_params_same_digest(self): + rf = RequestFactory() + a = rf.get('/', {'appId': '11111111-1111-1111-1111-111111111111'}) + b = rf.get('/', {'appId': '11111111-1111-1111-1111-111111111111'}) + self.assertEqual(_make_key('conferences.summary', a), _make_key('conferences.summary', b)) + + def test_make_key_different_app_different_digest(self): + rf = RequestFactory() + a = rf.get('/', {'appId': '11111111-1111-1111-1111-111111111111'}) + b = rf.get('/', {'appId': '22222222-2222-2222-2222-222222222222'}) + self.assertNotEqual(_make_key('conferences.summary', a), _make_key('conferences.summary', b)) + + def test_make_key_different_date_range_different_digest(self): + rf = RequestFactory() + a = rf.get( + '/', + { + 'appId': '11111111-1111-1111-1111-111111111111', + 'created_at_gte': '2026-01-01T00:00:00Z', + }, + ) + b = rf.get( + '/', + { + 'appId': '11111111-1111-1111-1111-111111111111', + 'created_at_gte': '2026-02-01T00:00:00Z', + }, + ) + self.assertNotEqual(_make_key('issues.summary', a), _make_key('issues.summary', b)) + + def test_make_key_ignores_query_params_not_in_cache_key_params(self): + rf = RequestFactory() + base = rf.get('/', {'appId': '11111111-1111-1111-1111-111111111111'}) + with_extra = rf.get( + '/', + { + 'appId': '11111111-1111-1111-1111-111111111111', + 'limit': '50', + 'offset': '10', + 'foo': 'bar', + }, + ) + self.assertEqual(_make_key('sessions.summary', base), _make_key('sessions.summary', with_extra)) + + def test_make_key_different_endpoint_different_digest(self): + rf = RequestFactory() + req = rf.get('/', {'appId': '11111111-1111-1111-1111-111111111111'}) + self.assertNotEqual( + _make_key('conferences.summary', req), + _make_key('issues.summary', req), + ) + + def test_cached_json_miss_then_hit_compute_once(self): + rf = RequestFactory() + req = rf.get('/', {'appId': '11111111-1111-1111-1111-111111111111'}) + calls = [] + + def compute(): + calls.append(1) + return {'data': [{'n': 1}]} + + p1, hit1 = cached_json('conferences.summary', req, compute) + p2, hit2 = cached_json('conferences.summary', req, compute) + + self.assertEqual(len(calls), 1) + self.assertFalse(hit1) + self.assertTrue(hit2) + self.assertEqual(p1, p2) + self.assertEqual(p1['data'][0]['n'], 1) + + @override_settings(SUMMARY_CACHE_TTL=42) + def test_get_ttl_reads_setting(self): + self.assertEqual(get_ttl(), 42) + + def test_make_key_includes_conference_id_when_present(self): + rf = RequestFactory() + a = rf.get('/', {'appId': '11111111-1111-1111-1111-111111111111'}) + b = rf.get( + '/', + { + 'appId': '11111111-1111-1111-1111-111111111111', + 'conferenceId': 'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa', + }, + ) + self.assertNotEqual(_make_key('conferences.summary', a), _make_key('conferences.summary', b)) + + def test_cached_json_cache_get_failure_still_returns_payload(self): + rf = RequestFactory() + req = rf.get('/', {'appId': '11111111-1111-1111-1111-111111111111'}) + + def compute(): + return {'ok': True} + + with patch('app.summary_cache.cache.get', side_effect=RuntimeError('redis down')): + with patch('app.summary_cache.cache.set', return_value=True): + payload, hit = cached_json('issues.summary', req, compute) + + self.assertFalse(hit) + self.assertEqual(payload, {'ok': True}) + + def test_cached_json_cache_set_failure_still_returns_payload(self): + rf = RequestFactory() + req = rf.get('/', {'appId': '11111111-1111-1111-1111-111111111111'}) + + def compute(): + return {'stored': False} + + with patch('app.summary_cache.cache.set', side_effect=RuntimeError('redis down')): + payload, hit = cached_json('connections.summary', req, compute) + + self.assertFalse(hit) + self.assertEqual(payload, {'stored': False}) From af9351d33782aaad6b67fd917a96070f12bb2fbf Mon Sep 17 00:00:00 2001 From: Samson Nkrumah Date: Mon, 27 Apr 2026 19:49:15 +0100 Subject: [PATCH 6/9] fix: bucket created_at_gte/created_at_lte to the minute in cache key MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The dashboard sends `new Date().toISOString()` minus 30 days as created_at_gte (web app.vue:189), which is millisecond-precise. With the unrounded value going straight into _make_key, every page load — even back-to-back reloads — produced a unique SHA1 digest and a fresh cache miss, so the warm path never served real users: flush redis -> 0 keys load dashboard -> 8 keys reload -> 16 keys (8 stale + 8 fresh) prewarm_summaries had the same problem on the write side: its own since_window = utcnow() - 30d advanced every run, so the entries it populated never matched what the dashboard requested. Truncate ISO timestamps to the minute (YYYY-MM-DDTHH:MM) before hashing, so two requests in the same wall-clock minute share an entry. Correctness still holds because the 60s TTL bounds staleness regardless of bucket size. After the fix, two same-minute dashboard loads both produce 8 keys (no growth), and the 2nd load's slow endpoints serve from cache: before after /v1/sessions/summary 2314ms -> 411ms /v1/connections/setup-time-summary 1117ms -> 229ms /v1/conferences/summary 994ms -> 48ms Two regression tests added covering the bucket and the minute boundary. --- app/summary_cache.py | 18 ++++++++++++++++++ app/tests/test_summary_cache.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/app/summary_cache.py b/app/summary_cache.py index 33940ff..a471c39 100644 --- a/app/summary_cache.py +++ b/app/summary_cache.py @@ -33,12 +33,30 @@ 'participantId', ) +# Params whose ISO timestamps should be truncated to the minute before hashing, +# so that the dashboard's millisecond-precise `now - 30d` doesn't produce a +# unique cache key per page load. Bucketing means two requests in the same +# wall-clock minute share an entry; correctness still holds because the cache +# entry's own TTL bounds staleness regardless of bucket size. +BUCKETED_PARAMS = ('created_at_gte', 'created_at_lte') + + +def _bucket_minute(value): + # ISO 8601: 2026-03-28T17:13:18.382Z -> 2026-03-28T17:13Z (first 16 chars + # are YYYY-MM-DDTHH:MM). Anything that doesn't match the layout is + # passed through unchanged so the key still differentiates malformed input. + if not value or len(value) < 16 or value[10] != 'T' or value[13] != ':': + return value + return value[:16] + 'Z' + def _make_key(endpoint, request): parts = [endpoint] for name in CACHE_KEY_PARAMS: val = request.GET.get(name) if val: + if name in BUCKETED_PARAMS: + val = _bucket_minute(val) parts.append(f'{name}={val}') raw = '|'.join(parts) # Keep key short but unique; include a readable prefix for ops visibility. diff --git a/app/tests/test_summary_cache.py b/app/tests/test_summary_cache.py index cb35100..2388ed8 100644 --- a/app/tests/test_summary_cache.py +++ b/app/tests/test_summary_cache.py @@ -51,6 +51,34 @@ def test_make_key_different_date_range_different_digest(self): ) self.assertNotEqual(_make_key('issues.summary', a), _make_key('issues.summary', b)) + def test_make_key_buckets_sub_minute_timestamp_precision(self): + # The dashboard sends `new Date().toISOString()` minus 30d, which is + # millisecond-precise. Two reloads seconds apart must collapse to the + # same key, otherwise prewarm cannot help and Redis fills with + # single-use entries. + rf = RequestFactory() + a = rf.get('/', { + 'appId': '11111111-1111-1111-1111-111111111111', + 'created_at_gte': '2026-03-28T17:13:18.382Z', + }) + b = rf.get('/', { + 'appId': '11111111-1111-1111-1111-111111111111', + 'created_at_gte': '2026-03-28T17:13:42.001Z', + }) + self.assertEqual(_make_key('sessions.summary', a), _make_key('sessions.summary', b)) + + def test_make_key_separates_distinct_minutes(self): + rf = RequestFactory() + a = rf.get('/', { + 'appId': '11111111-1111-1111-1111-111111111111', + 'created_at_gte': '2026-03-28T17:13:00Z', + }) + b = rf.get('/', { + 'appId': '11111111-1111-1111-1111-111111111111', + 'created_at_gte': '2026-03-28T17:14:00Z', + }) + self.assertNotEqual(_make_key('sessions.summary', a), _make_key('sessions.summary', b)) + def test_make_key_ignores_query_params_not_in_cache_key_params(self): rf = RequestFactory() base = rf.get('/', {'appId': '11111111-1111-1111-1111-111111111111'}) From f2c7aef4ce83ed9c1ebd7b5645789fe17b043628 Mon Sep 17 00:00:00 2001 From: agonza1 Date: Tue, 5 May 2026 19:30:32 -0400 Subject: [PATCH 7/9] Fix conference participant counts for dashboard and list Count only active non-SFU participants with a session on the conference, excluding peer stubs without sessions and SFU endpoints. Shared annotation in conference_query.py used by conferences list and participant-count summary. Co-authored-by: Cursor --- app/conference_query.py | 18 ++++++++++++++++++ ...onference_participant_count_summary_view.py | 16 ++-------------- app/views/conference_view.py | 14 +++----------- 3 files changed, 23 insertions(+), 25 deletions(-) create mode 100644 app/conference_query.py diff --git a/app/conference_query.py b/app/conference_query.py new file mode 100644 index 0000000..9ab0edf --- /dev/null +++ b/app/conference_query.py @@ -0,0 +1,18 @@ +""" +Query helpers for Conference-related aggregates. + +Peers created only for addConnection (P2P remote) are linked on conference.participants +but often have no Session and no name; they are excluded. SFU endpoints (is_sfu) are +excluded from participant totals — only human/clients with a session in the conference +are counted. +""" + +from django.db.models import Count, F, Q + +PARTICIPANTS_COUNT_ANNOTATION = Count( + 'participants', + filter=Q(participants__is_active=True) + & Q(participants__is_sfu=False) + & Q(participants__sessions__conference_id=F('id')), + distinct=True, +) diff --git a/app/views/conference_participant_count_summary_view.py b/app/views/conference_participant_count_summary_view.py index 188d916..6433c11 100644 --- a/app/views/conference_participant_count_summary_view.py +++ b/app/views/conference_participant_count_summary_view.py @@ -2,13 +2,11 @@ from collections import Counter from django.core.exceptions import ValidationError -from django.db.models import Count, IntegerField, OuterRef, Subquery - +from ..conference_query import PARTICIPANTS_COUNT_ANNOTATION from ..errors import INVALID_PARAMETERS, MISSING_PARAMETERS, PMError from ..summary_cache import cached_json from ..utils import JSONHttpResponse from ..models.conference import Conference -from ..models.participant import Participant from .generic_view import GenericView @@ -50,17 +48,7 @@ def compute(): try: per_conf = (Conference.objects .filter(**filters) - .annotate( - participants_count=Subquery( - Participant.objects.filter( - conferences=OuterRef('pk'), - is_active=True, - ).order_by().values('conferences').annotate( - cnt=Count('id', distinct=True) - ).values('cnt')[:1], - output_field=IntegerField(), - ), - ) + .annotate(participants_count=PARTICIPANTS_COUNT_ANNOTATION) .values_list('participants_count', flat=True)) except ValidationError: raise PMError(status=400, app_error=INVALID_PARAMETERS) diff --git a/app/views/conference_view.py b/app/views/conference_view.py index 722899e..a5c5a25 100644 --- a/app/views/conference_view.py +++ b/app/views/conference_view.py @@ -1,14 +1,14 @@ import datetime from django.core.exceptions import ValidationError -from django.db.models import Count, Exists, IntegerField, OuterRef, Subquery +from django.db.models import Exists, OuterRef +from ..conference_query import PARTICIPANTS_COUNT_ANNOTATION from ..errors import (INVALID_PARAMETERS, CONFERENCE_NOT_FOUND, MISSING_PARAMETERS, PMError) from ..utils import JSONHttpResponse, serialize, paginate_and_serialize from ..models.conference import Conference from ..models.issue import Issue -from ..models.participant import Participant from .generic_view import GenericView class ConferencesView(GenericView): @@ -63,15 +63,7 @@ def filter(cls, request): has_warnings=Exists( Issue.objects.filter(conference=OuterRef('pk'), type='w', is_active=True) ), - participants_count=Subquery( - Participant.objects.filter( - conferences=OuterRef('pk'), - is_active=True, - ).order_by().values('conferences').annotate( - cnt=Count('id', distinct=True) - ).values('cnt')[:1], - output_field=IntegerField(), - ), + participants_count=PARTICIPANTS_COUNT_ANNOTATION, ) if filter_by_issue_code: objs = objs.distinct() From f62b9751878e7195a83fce275b59f71111196248 Mon Sep 17 00:00:00 2001 From: agonza1 Date: Tue, 5 May 2026 19:55:07 -0400 Subject: [PATCH 8/9] Fix participants_count when filtering conferences by participantId Use a correlated Subquery instead of Count on the outer participants join. Filtering by participantId narrowed that join to one person, so the aggregate incorrectly showed 1 for multi-participant calls. The subquery counts independently. Add regression test. Co-authored-by: Cursor --- app/conference_query.py | 24 +++++++++---- app/tests/test_pr26_regressions.py | 35 +++++++++++++++++++ ...nference_participant_count_summary_view.py | 4 +-- app/views/conference_view.py | 4 +-- 4 files changed, 56 insertions(+), 11 deletions(-) diff --git a/app/conference_query.py b/app/conference_query.py index 9ab0edf..b661ede 100644 --- a/app/conference_query.py +++ b/app/conference_query.py @@ -5,14 +5,24 @@ but often have no Session and no name; they are excluded. SFU endpoints (is_sfu) are excluded from participant totals — only human/clients with a session in the conference are counted. + +participants_count must use a Subquery (not Count on the outer queryset's participants +relation): when /v1/conferences is filtered by participantId, the outer query joins +participants and would otherwise restrict the aggregate to that single participant. """ -from django.db.models import Count, F, Q +from django.db.models import Count, IntegerField, OuterRef, Subquery + +from .models.participant import Participant -PARTICIPANTS_COUNT_ANNOTATION = Count( - 'participants', - filter=Q(participants__is_active=True) - & Q(participants__is_sfu=False) - & Q(participants__sessions__conference_id=F('id')), - distinct=True, +PARTICIPANTS_COUNT_SUBQUERY = Subquery( + Participant.objects.filter( + conferences=OuterRef('pk'), + is_active=True, + is_sfu=False, + sessions__conference_id=OuterRef('pk'), + ).order_by().values('conferences').annotate( + cnt=Count('id', distinct=True), + ).values('cnt')[:1], + output_field=IntegerField(), ) diff --git a/app/tests/test_pr26_regressions.py b/app/tests/test_pr26_regressions.py index 7e47b65..e93af93 100644 --- a/app/tests/test_pr26_regressions.py +++ b/app/tests/test_pr26_regressions.py @@ -141,3 +141,38 @@ def test_gum_summary_skips_non_dict_issue_data(self): self.assertEqual(payload["total"], 1) self.assertEqual(payload["data"][0]["name"], "NotAllowedError") self.assertEqual(payload["data"][0]["count"], 1) + + def test_conferences_participant_id_filter_preserves_full_participants_count(self): + """Count must not collapse to 1 when the list is filtered to one participant.""" + conference = Conference.objects.create( + conference_id="conf-multi", + app=self.app, + ) + participants = [] + for i in range(3): + p = Participant.objects.create( + participant_id=f"user-{i}", + app=self.app, + ) + p.conferences.add(conference) + Session.objects.create( + conference=conference, + participant=p, + ) + participants.append(p) + + response = self.client.get( + "/v1/conferences", + { + "appId": str(self.app.id), + "participantId": str(participants[0].id), + "limit": "50", + }, + ) + + self.assertEqual(response.status_code, 200) + payload = json.loads(response.content) + self.assertEqual(payload["count"], 1) + self.assertEqual(len(payload["results"]), 1) + self.assertEqual(payload["results"][0]["id"], str(conference.id)) + self.assertEqual(payload["results"][0]["participants_count"], 3) diff --git a/app/views/conference_participant_count_summary_view.py b/app/views/conference_participant_count_summary_view.py index 6433c11..eed51f3 100644 --- a/app/views/conference_participant_count_summary_view.py +++ b/app/views/conference_participant_count_summary_view.py @@ -2,7 +2,7 @@ from collections import Counter from django.core.exceptions import ValidationError -from ..conference_query import PARTICIPANTS_COUNT_ANNOTATION +from ..conference_query import PARTICIPANTS_COUNT_SUBQUERY from ..errors import INVALID_PARAMETERS, MISSING_PARAMETERS, PMError from ..summary_cache import cached_json from ..utils import JSONHttpResponse @@ -48,7 +48,7 @@ def compute(): try: per_conf = (Conference.objects .filter(**filters) - .annotate(participants_count=PARTICIPANTS_COUNT_ANNOTATION) + .annotate(participants_count=PARTICIPANTS_COUNT_SUBQUERY) .values_list('participants_count', flat=True)) except ValidationError: raise PMError(status=400, app_error=INVALID_PARAMETERS) diff --git a/app/views/conference_view.py b/app/views/conference_view.py index a5c5a25..6c02fe3 100644 --- a/app/views/conference_view.py +++ b/app/views/conference_view.py @@ -3,7 +3,7 @@ from django.core.exceptions import ValidationError from django.db.models import Exists, OuterRef -from ..conference_query import PARTICIPANTS_COUNT_ANNOTATION +from ..conference_query import PARTICIPANTS_COUNT_SUBQUERY from ..errors import (INVALID_PARAMETERS, CONFERENCE_NOT_FOUND, MISSING_PARAMETERS, PMError) from ..utils import JSONHttpResponse, serialize, paginate_and_serialize @@ -63,7 +63,7 @@ def filter(cls, request): has_warnings=Exists( Issue.objects.filter(conference=OuterRef('pk'), type='w', is_active=True) ), - participants_count=PARTICIPANTS_COUNT_ANNOTATION, + participants_count=PARTICIPANTS_COUNT_SUBQUERY, ) if filter_by_issue_code: objs = objs.distinct() From d306fe40fd3e1553e8b19d94cc0feb42298049d2 Mon Sep 17 00:00:00 2001 From: agonza1 Date: Tue, 5 May 2026 19:59:18 -0400 Subject: [PATCH 9/9] Clarify participantId filter regression test docstring Co-authored-by: Cursor --- app/tests/test_pr26_regressions.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/app/tests/test_pr26_regressions.py b/app/tests/test_pr26_regressions.py index e93af93..4106285 100644 --- a/app/tests/test_pr26_regressions.py +++ b/app/tests/test_pr26_regressions.py @@ -143,7 +143,10 @@ def test_gum_summary_skips_non_dict_issue_data(self): self.assertEqual(payload["data"][0]["count"], 1) def test_conferences_participant_id_filter_preserves_full_participants_count(self): - """Count must not collapse to 1 when the list is filtered to one participant.""" + """ + When the conference list is filtered to one person (participantId), + participants_count must still reflect everyone in that conference, not 1. + """ conference = Conference.objects.create( conference_id="conf-multi", app=self.app,