feat: adding support for basic utm tags (#13)

api/src/app.ts

@@ -15,6 +15,7 @@ import {
   getUniqueSessionsByCountryHandler,
   getCountSessionsByUserAgentHandler,
   getStatsHandler,
+  getCountSessionsByUtmHandler,
 } from "./routes/counts";
 import {
   healthCheckHandler,
@@ -57,6 +58,7 @@ export function getApp() {
   app.get("/admin/api/top_referrers", authMiddleware, getTopReferrersHandler);
   app.get("/admin/api/unique_sessions_by_country", authMiddleware, getUniqueSessionsByCountryHandler);
   app.get("/admin/api/count_sessions_by_user_agent", authMiddleware, getCountSessionsByUserAgentHandler);
+  app.get("/admin/api/count_sessions_by_utm", authMiddleware, getCountSessionsByUtmHandler);
   app.get("/admin/api/stats", authMiddleware, getStatsHandler);
   app.get("/admin/api/config", authMiddleware, siteConfigHandler);
   app.get("/admin/api/has-events", authMiddleware, hasEventsHandler);

api/src/migrations/libsql/005_utm.ts

@@ -0,0 +1,17 @@
+import { getLibsqlRepo } from "../../repository/repo";
+
+async function migrate(): Promise<boolean> {
+  const db = getLibsqlRepo().db();
+
+  await db.executeMultiple(`
+    ALTER TABLE events ADD COLUMN utm_source TEXT;
+    ALTER TABLE events ADD COLUMN utm_medium TEXT;
+    ALTER TABLE events ADD COLUMN utm_campaign TEXT;
+    ALTER TABLE events ADD COLUMN utm_term TEXT;
+    ALTER TABLE events ADD COLUMN utm_content TEXT;
+  `);
+
+  return true;
+}
+
+export default migrate;

api/src/migrations/libsql/index.ts

@@ -2,3 +2,4 @@ export * as migation_001_basic_schema from "./001_basic_schema";
 export * as migation_002_indexes from "./002_indexes";
 export * as migation_003_data_io from "./003_data_io";
 export * as migation_004_apps from "./004_apps";
+export * as migation_005_utm from "./005_utm";

api/src/migrations/postgres/005_utm.ts

@@ -0,0 +1,18 @@
+import { getPostgresRepo } from "../../repository/repo";
+
+async function migrate(): Promise<boolean> {
+  const sql = getPostgresRepo().db();
+
+  await sql`
+    ALTER TABLE events
+    ADD COLUMN IF NOT EXISTS utm_source TEXT,
+    ADD COLUMN IF NOT EXISTS utm_medium TEXT,
+    ADD COLUMN IF NOT EXISTS utm_campaign TEXT,
+    ADD COLUMN IF NOT EXISTS utm_term TEXT,
+    ADD COLUMN IF NOT EXISTS utm_content TEXT;
+  `;
+
+  return true;
+}
+
+export default migrate;

api/src/migrations/postgres/index.ts

@@ -2,3 +2,4 @@ export * as migation_001_basic_schema from "./001_basic_schema";
 export * as migation_002_indexes from "./002_indexes";
 export * as migation_003_data_io from "./003_data_io";
 export * as migation_004_apps from "./004_apps";
+export * as migation_005_utm from "./005_utm";

api/src/migrations/timescale/005_utm.ts

@@ -0,0 +1,18 @@
+import { getTimescaleRepo } from "../../repository/repo";
+
+async function migrate(): Promise<boolean> {
+  const sql = getTimescaleRepo().db();
+
+  await sql`
+    ALTER TABLE events
+    ADD COLUMN IF NOT EXISTS utm_source TEXT,
+    ADD COLUMN IF NOT EXISTS utm_medium TEXT,
+    ADD COLUMN IF NOT EXISTS utm_campaign TEXT,
+    ADD COLUMN IF NOT EXISTS utm_term TEXT,
+    ADD COLUMN IF NOT EXISTS utm_content TEXT;
+  `;
+
+  return true;
+}
+
+export default migrate;

api/src/migrations/timescale/index.ts

@@ -2,3 +2,4 @@ export * as migation_001_basic_schema from "./001_basic_schema";
 export * as migation_002_data_io from "./002_data_io";
 export * as migation_003_apps from "./003_apps";
 export * as migation_004_indexes from "./004_indexes";
+export * as migation_005_utm from "./005_utm";

api/src/repository/libsql.ts

@@ -8,6 +8,7 @@ import {
   SessionItem,
   Stats,
   UserAgentQueryKeys,
+  UtmTagKey,
 } from "@shared/types";
 import { WebEvent } from "../types/models";
 import { webEventToSqlFormat } from "../utils/parsers";
@@ -23,9 +24,9 @@ export class LibsqlRepo implements IDataRepo {
   private uid: ShortUniqueId;
   private client: LibsqlClient;
   public static allColumns =
-    "session_id,path,timestamp,ip,user_agent,referrer,language,country,screen_width,screen_height,window_width,window_height,bot_name,bot_category,bot_url,bot_producer_name,bot_producer_url,client_type,client_name,client_version,client_engine,client_engine_version,device_type,device_brand,device_model,os_name,os_version,os_platform";
+    "session_id,path,timestamp,ip,user_agent,referrer,language,country,screen_width,screen_height,window_width,window_height,bot_name,bot_category,bot_url,bot_producer_name,bot_producer_url,client_type,client_name,client_version,client_engine,client_engine_version,device_type,device_brand,device_model,os_name,os_version,os_platform,app_id,utm_source,utm_medium,utm_campaign,utm_term,utm_content";
   public static allColumnsValues =
-    ":session_id,:path,:timestamp,:ip,:user_agent,:referrer,:language,:country,:screen_width,:screen_height,:window_width,:window_height,:bot_name,:bot_category,:bot_url,:bot_producer_name,:bot_producer_url,:client_type,:client_name,:client_version,:client_engine,:client_engine_version,:device_type,:device_brand,:device_model,:os_name,:os_version,:os_platform";
+    ":session_id,:path,:timestamp,:ip,:user_agent,:referrer,:language,:country,:screen_width,:screen_height,:window_width,:window_height,:bot_name,:bot_category,:bot_url,:bot_producer_name,:bot_producer_url,:client_type,:client_name,:client_version,:client_engine,:client_engine_version,:device_type,:device_brand,:device_model,:os_name,:os_version,:os_platform,:app_id,:utm_source,:utm_medium,:utm_campaign,:utm_term,:utm_content";
 
   constructor() {
     this.uid = new ShortUniqueId({ length: 7 });
@@ -212,6 +213,26 @@ export class LibsqlRepo implements IDataRepo {
     return result.rows as unknown as CountByKeyValue[];
   }
 
+  async getSessionCountByUtmTag(appId: string, key: UtmTagKey, numberOfDays = 30) {
+    const result = await this.client.execute({
+      sql: `
+        SELECT '${key}' as "key", ${key} as value, COUNT(DISTINCT session_id) as count
+        FROM events
+        WHERE timestamp >= :start
+        AND app_id = :appId
+        GROUP BY :key
+        ORDER BY count DESC, value ASC
+      `,
+      args: {
+        appId,
+        start: getDaysAgo(numberOfDays).toISOString(),
+        key,
+      },
+    });
+
+    return result.rows as unknown as CountByKeyValue[];
+  }
+
   async hasAnyEvents() {
     const result = await this.client.execute("select * from events limit 1");
     return result.rows.length > 0;

api/src/repository/mongo.ts

@@ -7,6 +7,7 @@ import {
   SessionItem,
   Stats,
   UserAgentQueryKeys,
+  UtmTagKey,
 } from "@shared/types";
 import { MongoClient } from "mongodb";
 import { WebEvent } from "../types/models";
@@ -59,9 +60,46 @@ export class MongoRepo implements IDataRepo {
         {
           $project: {
             _id: 0,
+            key: { $literal: key },
             value: 1,
             count: { $size: "$count" },
+          },
+        },
+        {
+          $sort: {
+            count: -1,
+            value: 1,
+          },
+        },
+      ])
+      .toArray();
+
+    return results;
+  }
+
+  async getSessionCountByUtmTag(app_id: string, key: UtmTagKey, numberOfDays = 30) {
+    const results = await this.db()
+      .collection(cols.events)
+      .aggregate<CountByKeyValue>([
+        {
+          $match: {
+            timestamp: { $gte: getDaysAgo(numberOfDays) },
+            app_id,
+          },
+        },
+        {
+          $group: {
+            _id: `$${key}`,
+            value: { $first: `$${key}` },
+            count: { $addToSet: "$session_id" },
+          },
+        },
+        {
+          $project: {
+            _id: 0,
             key: { $literal: key },
+            value: 1,
+            count: { $size: "$count" },
           },
         },
         {

api/src/repository/postgres.ts

@@ -7,6 +7,7 @@ import {
   SessionItem,
   Stats,
   UserAgentQueryKeys,
+  UtmTagKey,
 } from "@shared/types";
 import fs from "fs/promises";
 import postgres from "postgres";
@@ -49,7 +50,18 @@ export class PostgresRepo implements IDataRepo {
     this.sql = postgres(url, options);
   }
 
-  getSessionCountByUserAgent(appId: string, key: UserAgentQueryKeys, numberOfDays = 30): Promise<CountByKeyValue[]> {
+  getSessionCountByUserAgent(appId: string, key: UserAgentQueryKeys, numberOfDays = 30) {
+    return this.sql<CountByKeyValue[]>`
+      SELECT ${key} as key, ${this.sql(key)} as value, COUNT(DISTINCT session_id) as count
+      FROM events
+      WHERE timestamp >= ${getDaysAgo(numberOfDays).toISOString()}
+      AND app_id = ${appId}
+      GROUP BY ${this.sql(key)}
+      ORDER BY count DESC, value ASC
+    `;
+  }
+
+  getSessionCountByUtmTag(appId: string, key: UtmTagKey, numberOfDays = 30) {
     return this.sql<CountByKeyValue[]>`
       SELECT ${key} as key, ${this.sql(key)} as value, COUNT(DISTINCT session_id) as count
       FROM events

api/src/repository/types.ts

@@ -7,6 +7,7 @@ import {
   SessionItem,
   Stats,
   UserAgentQueryKeys,
+  UtmTagKey,
 } from "@shared/types";
 import { WebEvent } from "../types/models";
 import { App } from "@shared/app";
@@ -27,6 +28,7 @@ export interface IDataRepo {
   getUniqueSessionsByCountry(appId: string, numberOfDays?: number): Promise<CountByCountry[]>;
   getStats(appId: string, numberOfDays?: number): Promise<Stats>;
   getSessionCountByUserAgent(appId: string, key: UserAgentQueryKeys, numberOfDays?: number): Promise<CountByKeyValue[]>;
+  getSessionCountByUtmTag(appId: string, key: UtmTagKey, numberOfDays?: number): Promise<CountByKeyValue[]>;
 
   // Exports
   startExport(): Promise<void>;

api/src/routes/counts.ts

@@ -1,7 +1,7 @@
 import { UserAgentQueryKeys } from "@shared/types";
 import { Request, Response } from "express";
 import { getDataRepo } from "../repository/repo";
-import { isUserAgentQueryKey } from "../utils/guards";
+import { isUserAgentQueryKey, isUtmTagKey } from "../utils/guards";
 import { parseDays, parseAppId } from "../utils/queryParsers";
 
 export async function getSessionsPerDayHandler(req: Request, res: Response) {
@@ -101,3 +101,21 @@ export async function getCountSessionsByUserAgentHandler(
     res.status(500).json({ message: "Internal server error" });
   }
 }
+
+export async function getCountSessionsByUtmHandler(req: Request<any, any, { key: UserAgentQueryKeys }>, res: Response) {
+  try {
+    const { key } = req.query;
+    if (!isUtmTagKey(key)) {
+      throw new Error("Invalid key. Must be one of utm_source, utm_medium, utm_campaign, utm_term, utm_content");
+    }
+
+    const numberOfDays = parseDays(req);
+    const appId = parseAppId(req);
+
+    const result = await getDataRepo().getSessionCountByUtmTag(appId, key, numberOfDays);
+    res.json(result);
+  } catch (err) {
+    console.error(err);
+    res.status(500).json({ message: "Internal server error" });
+  }
+}

api/src/scripts/seedDemo.ts

@@ -7,7 +7,13 @@ import { TimescaleRepo } from "../repository/timescale";
 import { WebEvent } from "../types/models";
 import { getDaysAgoRandomTime } from "../utils/date";
 import { webEventToSqlFormat } from "../utils/parsers";
-import { generateUsers, getRandomPath, getRandomReferrer, getRandomScreenSize } from "./seedDemoData";
+import {
+  generateRandomUtm,
+  generateUsers,
+  getRandomPath,
+  getRandomReferrer,
+  getRandomScreenSize,
+} from "./seedDemoData";
 
 export async function seedDemo(logs = true, userCount = 1000, daysCount = 365) {
   const logger = logs ? console.log : () => {};
@@ -51,6 +57,7 @@ export async function seedDemo(logs = true, userCount = 1000, daysCount = 365) {
           window_height: user.screen_size_temp.height,
           window: getRandomScreenSize(),
           referrer: getRandomReferrer(),
+          ...generateRandomUtm(),
         };
 
         // @ts-ignore

api/src/scripts/seedDemoData.ts

@@ -191,6 +191,22 @@ function generateRandomLanguage() {
   return languages[Math.floor(Math.random() * languages.length)];
 }
 
+const utm_sources = ["google", "facebook", "twitter"];
+const utm_mediums = ["cpc", "social", "email"];
+const utm_campaigns = ["awesome-campaign", "awesome-campaign-2", "awesome-campaign-3"];
+
+export function generateRandomUtm() {
+  // 99% return null
+  if (Math.random() < 0.99) {
+    return {};
+  }
+  return {
+    utm_source: utm_sources[Math.floor(Math.random() * utm_sources.length)],
+    utm_medium: utm_mediums[Math.floor(Math.random() * utm_mediums.length)],
+    utm_campaign: utm_campaigns[Math.floor(Math.random() * utm_campaigns.length)],
+  };
+}
+
 function generateRandomIP(): string {
   const ipParts = [];
 

api/src/tests/api.spec.ts

@@ -261,6 +261,18 @@ async function apiDataQueries() {
     expect(item.key).toBeDefined();
     expect(item.key).toEqual("os_name");
   }
+
+  const countByUtmTagSource = await supertest(app)
+    .get(`/admin/api/count_sessions_by_utm?key=utm_source&app_id=${app_id}`)
+    .auth(user, pass)
+    .expect(200);
+
+  for (const item of countByUtmTagSource.body) {
+    expect(item.count).toBeDefined();
+    expect(item.value).toBeDefined();
+    expect(item.key).toBeDefined();
+    expect(item.key).toEqual("utm_source");
+  }
 }
 
 async function apiAppsCrud() {

api/src/types/models.ts

@@ -24,7 +24,8 @@ export type WebEvent = {
   screen_height?: number | undefined;
   window_width?: number | undefined;
   window_height?: number | undefined;
-} & EventClientDetails;
+} & EventClientDetails &
+  UtmTags;
 
 export type EventClientDetails = {
   bot_name?: string | undefined;
@@ -44,3 +45,11 @@ export type EventClientDetails = {
   os_version?: string | undefined;
   os_platform?: OperatingSystemResult["platform"] | undefined;
 };
+
+export type UtmTags = {
+  utm_source?: string | undefined;
+  utm_medium?: string | undefined;
+  utm_campaign?: string | undefined;
+  utm_term?: string | undefined;
+  utm_content?: string | undefined;
+};

api/src/utils/extractEvent.ts

@@ -10,14 +10,19 @@ export function extractEvent(request: Request): WebEvent {
     session_id: extractSessionId(request),
     app_id: request.body.app_id,
     path: request.body.path,
-    referrer: request.body.referrer,
-    screen_width: request.body.screen_width,
-    screen_height: request.body.screen_height,
-    window_width: request.body.window_width,
-    window_height: request.body.window_height,
-    user_agent: request.headers["user-agent"] as string | null,
-    language: request.headers["accept-language"] as string | null,
-    country: request.headers["cf-ipcountry"] as string | null,
+    referrer: request.body.referrer || null,
+    screen_width: request.body.screen_width || null,
+    screen_height: request.body.screen_height || null,
+    window_width: request.body.window_width || null,
+    window_height: request.body.window_height || null,
+    user_agent: (request.headers["user-agent"] as string | null) || null,
+    language: (request.headers["accept-language"] as string | null) || null,
+    country: (request.headers["cf-ipcountry"] as string | null) || null,
+    utm_source: request.body.utm_source || null,
+    utm_medium: request.body.utm_medium || null,
+    utm_campaign: request.body.utm_campaign || null,
+    utm_term: request.body.utm_term || null,
+    utm_content: request.body.utm_content || null,
     timestamp: new Date(),
   };
 

api/src/utils/guards.ts

@@ -1,4 +1,4 @@
-import { UserAgentQueryKeys } from "@shared/types";
+import { UserAgentQueryKeys, UtmTagKey } from "@shared/types";
 
 export function isUserAgentQueryKey(key: any): key is UserAgentQueryKeys {
   if (typeof key !== "string") false;
@@ -8,3 +8,12 @@ export function isUserAgentQueryKey(key: any): key is UserAgentQueryKeys {
 
   return true;
 }
+
+export function isUtmTagKey(key: any): key is UtmTagKey {
+  if (typeof key !== "string") false;
+  if (!["utm_source", "utm_medium", "utm_campaign", "utm_term", "utm_content"].includes(key)) {
+    return false;
+  }
+
+  return true;
+}