Speed up player event fetching

src/migrations/clickhouse/007AddEventPropsEventIdIndex.ts

@@ -0,0 +1 @@
+export const AddEventPropsEventIdIndex = `ALTER TABLE ${process.env.CLICKHOUSE_DB}.event_props ADD INDEX IF NOT EXISTS event_id_idx (event_id) TYPE minmax GRANULARITY 64;`

src/migrations/clickhouse/index.ts

@@ -7,6 +7,7 @@ import { formatDateForClickHouse } from '../../lib/clickhouse/formatDateTime'
 import { CreatePlayerGameStatSnapshotsTable } from './004CreatePlayerGameStatSnapshotsTable'
 import { MigrateEventsTimestampsToDate64 } from './005MigrateEventsTimestampsToDate64'
 import { CreatePlayerSessionsTable } from './006CreatePlayerSessionsTable'
+import { AddEventPropsEventIdIndex } from './007AddEventPropsEventIdIndex'
 
 type ClickHouseMigration = {
   name: string
@@ -37,6 +38,10 @@ const migrations: ClickHouseMigration[] = [
   {
     name: 'CreatePlayerSessionsTable',
     sql: CreatePlayerSessionsTable
+  },
+  {
+    name: 'AddEventPropsEventIdIndex',
+    sql: AddEventPropsEventIdIndex
   }
 ]
 

src/services/player.service.ts

@@ -314,46 +314,47 @@ export default class PlayerService extends Service {
     const em: EntityManager = req.ctx.em
     const clickhouse: ClickHouseClient = req.ctx.clickhouse
 
-    const aliases = player.aliases.getItems().map((alias) => alias.id).join(',')
+    const searchQuery = search
+      ? 'AND (e.name ILIKE {search: String} OR e.id IN (SELECT event_id FROM event_props WHERE prop_value ILIKE {search: String}))'
+      : ''
 
-    const searchQuery = search ? 'AND (name ILIKE {search: String} OR prop_value ILIKE {search: String})' : ''
-    const baseQuery = `FROM events
-      LEFT JOIN event_props ON events.id = event_props.event_id
-      WHERE player_alias_id IN (${aliases})
+    const baseQuery = `FROM events e
+      WHERE e.player_alias_id IN ({aliasIds:Array(UInt32)})
         ${searchQuery}`
 
     const query = `
-      SELECT DISTINCT events.*
-      ${baseQuery}
+      WITH filtered_events AS (
+        SELECT e.*
+        ${baseQuery}
+      )
+      SELECT 
+        *,
+        count(*) OVER() as total_count
+      FROM filtered_events
       ORDER BY created_at DESC
       LIMIT ${itemsPerPage}
       OFFSET ${Number(page) * itemsPerPage}
     `
 
-    const queryParams = { search: `%${search}%` }
+    const queryParams = {
+      search: `%${search}%`,
+      aliasIds: player.aliases.getItems().map((alias) => alias.id)
+    }
 
-    const items = await clickhouse.query({
+    const results = await clickhouse.query({
       query,
       query_params: queryParams,
       format: 'JSONEachRow'
-    }).then((res) => res.json<ClickHouseEvent>())
-    const events = await Event.massHydrate(em, items, clickhouse, true)
+    }).then((res) => res.json<ClickHouseEvent & { total_count: string }>())
 
-    const countQuery = `
-      SELECT count(DISTINCT events.id) AS count
-      ${baseQuery}`
-
-    const count = await clickhouse.query({
-      query: countQuery,
-      query_params: queryParams,
-      format: 'JSONEachRow'
-    }).then((res) => res.json<{ count: string }>())
+    const events = await Event.massHydrate(em, results, clickhouse, true)
+    const count = results.length > 0 ? Number(results[0].total_count) : 0
 
     return {
       status: 200,
       body: {
         events,
-        count: Number(count[0].count),
+        count,
         itemsPerPage
       }
     }