Speed up events endpoint

posthog/api/event.py

@@ -1,17 +1,17 @@
-from datetime import datetime
+from datetime import datetime, timedelta
 from posthog.models import Event, Person, Element, Action, ElementGroup, Filter, PersonDistinctId, Team
 from posthog.utils import friendly_time, request_to_date_query, append_data, convert_property_value, get_compare_period_dates, dict_from_cursor_fetchall
 from rest_framework import request, response, serializers, viewsets
 from rest_framework.decorators import action
 from django.db.models import QuerySet, F, Prefetch, Q
 from django.db.models.functions import Lag
-from django.db import connection
 from django.db.models.expressions import Window
+from django.db import connection
+from django.utils.timezone import now
 from typing import Any, Dict, List, Union
-import json
 from django.utils.timezone import now
+import json
 import pandas as pd
-from django.db import connection
 
 class ElementSerializer(serializers.ModelSerializer):
     event = serializers.CharField()
@@ -123,10 +123,29 @@ def _prefetch_events(self, events: List[Event]) -> List[Event]:
         return events
 
     def list(self, request: request.Request, *args: Any, **kwargs: Any) -> response.Response:
-        events = [event for event in self.get_queryset()[0: 101]]
-        prefetched_events = self._prefetch_events(events[0:100])
+        queryset = self.get_queryset()
+        monday = now() + timedelta(days=-now().weekday())
+        events = queryset.filter(timestamp__gte=monday.replace(hour=0, minute=0, second=0))[0: 101]
+
+        if len(events) < 101:
+            events = queryset[0: 101]
+
+        prefetched_events = self._prefetch_events([event for event in events])
+        path = request.get_full_path()
+
+        reverse = request.GET.get('orderBy', '-timestamp') != '-timestamp'
+        if len(events) > 100:
+            next_url: Union[bool, str] = '{}{}{}={}'.format(
+                path,
+                '&' if '?' in path else '?',
+                'after' if reverse else 'before',
+                events[99].timestamp.strftime('%Y-%m-%dT%H:%M:%S.%fZ')
+            )
+        else:
+            next_url = False
+
         return response.Response({
-            'next': len(events) > 100,
+            'next': next_url,
             'results': EventSerializer(prefetched_events, many=True).data
         })
 

posthog/api/test/test_event.py

@@ -17,15 +17,15 @@ def test_filter_events(self):
         Event.objects.create(team=self.team, distinct_id='some-random-uid', properties={"$ip": '8.8.8.8'})
         Event.objects.create(team=self.team, distinct_id='some-other-one', properties={"$ip": '8.8.8.8'})
 
-        with self.assertNumQueries(10):
+        with self.assertNumQueries(11):
             response = self.client.get('/api/event/?distinct_id=2').json()
         self.assertEqual(response['results'][0]['person'], 'tim@posthog.com')
         self.assertEqual(response['results'][0]['elements'][0]['tag_name'], 'button')
 
     def test_filter_events_by_event_name(self):
         person = Person.objects.create(properties={'email': 'tim@posthog.com'}, team=self.team, distinct_ids=["2", 'some-random-uid'])
         event1 = Event.objects.create(event='event_name',team=self.team, distinct_id="2", properties={"$ip": '8.8.8.8'})
-        with self.assertNumQueries(7):
+        with self.assertNumQueries(8):
             response = self.client.get('/api/event/?event=event_name').json()
         self.assertEqual(response['results'][0]['event'], 'event_name')
 
@@ -34,7 +34,7 @@ def test_filter_events_by_properties(self):
         Event.objects.create(event='event_name',team=self.team, distinct_id="2", properties={"$browser": 'Chrome'})
         event2 = Event.objects.create(event='event_name',team=self.team, distinct_id="2", properties={"$browser": 'Safari'})
 
-        with self.assertNumQueries(7):
+        with self.assertNumQueries(8):
             response = self.client.get('/api/event/?properties=%s' % (json.dumps([{'key': '$browser', 'value': 'Safari'}]))).json()
         self.assertEqual(response['results'][0]['id'], event2.pk)
 
@@ -260,4 +260,15 @@ def test_sessions_count_buckets(self):
             if item['label'] == '30-60 minutes' or item['label'] == '3-10 seconds':
                 self.assertEqual(item['count'], 2)
             else:
-                self.assertEqual(item['count'], 1)
+                self.assertEqual(item['count'], 1)
+
+    def test_pagination(self):
+        events = []
+        for index in range(0, 150):
+            events.append(Event(team=self.team, event='some event', distinct_id='1'))
+        Event.objects.bulk_create(events)
+        response = self.client.get('/api/event/?distinct_id=1').json()
+        self.assertIn('distinct_id=1', response['next'])
+
+        page2 = self.client.get(response['next']).json()
+        self.assertEqual(len(page2['results']), 50)