feat: migrate leafwatch to use clickhouse client

apps/web/src/components/StaffTools/Panels/Profile/ProfileDetails.tsx

@@ -6,7 +6,7 @@ import {
   MapPinIcon
 } from '@heroicons/react/24/outline';
 import { AdjustmentsVerticalIcon } from '@heroicons/react/24/solid';
-import { LEAFWATCH_WORKER_URL, STATS_WORKER_URL } from '@hey/data/constants';
+import { STATS_WORKER_URL } from '@hey/data/constants';
 import type { Profile } from '@hey/lens';
 import humanize from '@hey/lib/humanize';
 import { useQuery } from '@tanstack/react-query';
@@ -30,7 +30,7 @@ const ProfileDetails: FC<ProfileDetailsProps> = ({ profile }) => {
     version: string;
   } | null> => {
     try {
-      const response = await axios.get(`${LEAFWATCH_WORKER_URL}/profile`, {
+      const response = await axios.get(`${STATS_WORKER_URL}/profileDetails`, {
         params: { id: profile.id }
       });
       const { data } = response;

packages/workers/leafwatch/.dev.vars.example

@@ -1,2 +1,2 @@
-CLICKHOUSE_REST_ENDPOINT=""
+CLICKHOUSE_PASSWORD=""
 IPAPI_KEY=""

packages/workers/leafwatch/package.json

@@ -13,6 +13,7 @@
     "worker:deploy": "wrangler deploy --var RELEASE:\"$(git rev-parse HEAD)\""
   },
   "dependencies": {
+    "@hey/clickhouse": "workspace:*",
     "@hey/data": "workspace:*",
     "@hey/lib": "workspace:*",
     "itty-router": "^4.0.23",

packages/workers/leafwatch/src/handlers/ingest.ts

@@ -1,3 +1,4 @@
+import createClickhouseClient from '@hey/clickhouse/createClickhouseClient';
 import { Errors } from '@hey/data/errors';
 import { ALL_EVENTS } from '@hey/data/tracking';
 import response from '@hey/lib/response';
@@ -77,58 +78,34 @@ export default async (request: WorkerRequest) => {
     const utmTerm = parsedUrl.searchParams.get('utm_term') || null;
     const utmContent = parsedUrl.searchParams.get('utm_content') || null;
 
-    const clickhouseResponse = await fetch(
-      request.env.CLICKHOUSE_REST_ENDPOINT,
-      {
-        method: 'POST',
-        headers: { 'Content-Type': 'application/json' },
-        body: `
-        INSERT INTO events (
+    const client = createClickhouseClient(request.env.CLICKHOUSE_PASSWORD);
+    const result = await client.insert({
+      table: 'events',
+      values: [
+        {
           name,
-          actor,
-          properties,
-          url,
-          city,
-          country,
-          region,
-          referrer,
-          platform,
-          browser,
-          browser_version,
-          os,
-          utm_source,
-          utm_medium,
-          utm_campaign,
-          utm_term,
-          utm_content
-        ) VALUES (
-          '${name}',
-          ${actor ? `'${actor}'` : null},
-          ${properties ? `'${JSON.stringify(properties)}'` : null},
-          ${url ? `'${url}'` : null},
-          ${ipData?.city ? `'${ipData?.city}'` : null},
-          ${ipData?.country ? `'${ipData?.country}'` : null},
-          ${ipData?.regionName ? `'${ipData?.regionName}'` : null},
-          ${referrer ? `'${referrer}'` : null},
-          ${platform ? `'${platform}'` : null},
-          ${ua.browser.name ? `'${ua.browser.name}'` : null},
-          ${ua.browser.version ? `'${ua.os.version}'` : null},
-          ${ua.os.name ? `'${ua.os.name}'` : null},
-          ${utmSource ? `'${utmSource}'` : null},
-          ${utmMedium ? `'${utmMedium}'` : null},
-          ${utmCampaign ? `'${utmCampaign}'` : null},
-          ${utmTerm ? `'${utmTerm}'` : null},
-          ${utmContent ? `'${utmContent}'` : null}
-        )
-      `
-      }
-    );
+          actor: actor || null,
+          properties: properties || null,
+          url: url || null,
+          city: ipData?.city || null,
+          country: ipData?.country || null,
+          region: ipData?.regionName || null,
+          referrer: referrer || null,
+          platform: platform || null,
+          browser: ua.browser.name || null,
+          browser_version: ua.browser.version || null,
+          os: ua.os.name || null,
+          utm_source: utmSource || null,
+          utm_medium: utmMedium || null,
+          utm_campaign: utmCampaign || null,
+          utm_term: utmTerm || null,
+          utm_content: utmContent || null
+        }
+      ],
+      format: 'JSONEachRow'
+    });
 
-    if (clickhouseResponse.status !== 200) {
-      return response({ success: false, error: Errors.StatusCodeIsNot200 });
-    }
-
-    return response({ success: true });
+    return response({ success: true, id: result.query_id });
   } catch (error) {
     throw error;
   }

packages/workers/leafwatch/src/index.ts

@@ -3,7 +3,6 @@ import response from '@hey/lib/response';
 import { createCors, error, Router, status } from 'itty-router';
 
 import ingest from './handlers/ingest';
-import profileDetails from './handlers/profileDetails';
 import buildRequest from './helpers/buildRequest';
 import type { Env, WorkerRequest } from './types';
 
@@ -23,7 +22,6 @@ router
       version: request.env.RELEASE ?? 'unknown'
     })
   )
-  .get('/profile', profileDetails)
   .post('/ingest', ingest)
   .all('*', () => error(404));
 

packages/workers/leafwatch/src/types.ts

@@ -2,7 +2,7 @@ import type { IRequestStrict } from 'itty-router';
 
 export interface Env {
   RELEASE: string;
-  CLICKHOUSE_REST_ENDPOINT: string;
+  CLICKHOUSE_PASSWORD: string;
   IPAPI_KEY: string;
 }
 

packages/workers/leafwatch/wrangler.toml

@@ -10,5 +10,5 @@ routes = [
 
 [env.production.vars]
 RELEASE = ""
-CLICKHOUSE_REST_ENDPOINT = ""
+CLICKHOUSE_PASSWORD = ""
 IPAPI_KEY = ""

packages/workers/stats/src/handlers/profileDetails.ts

@@ -0,0 +1,79 @@
+import createClickhouseClient from '@hey/clickhouse/createClickhouseClient';
+import { Errors } from '@hey/data/errors';
+import response from '@hey/lib/response';
+
+import type { WorkerRequest } from '../../../leafwatch/src/types';
+
+export default async (request: WorkerRequest) => {
+  const id = request.query.id as string;
+
+  if (!id) {
+    return response({ success: false, error: Errors.NoBody });
+  }
+
+  try {
+    const client = createClickhouseClient(request.env.CLICKHOUSE_PASSWORD);
+    const rows = await client.query({
+      query: `
+        WITH events_counts AS (
+          SELECT
+            actor,
+            country,
+            region,
+            city,
+            os,
+            browser,
+            browser_version,
+            COUNT() AS cnt
+          FROM events
+          WHERE actor = '${id}'
+          GROUP BY actor, country, region, city, os, browser, browser_version
+        )
+        SELECT
+          actor,
+          argMax(country, cnt) AS most_common_country,
+          argMax(region, cnt) AS most_common_region,
+          argMax(city, cnt) AS most_common_city,
+          SUM(cnt) AS number_of_events,
+          argMax(os, cnt) AS most_common_os,
+          argMax(browser, cnt) AS most_common_browser,
+          argMax(browser_version, cnt) AS most_common_browser_version
+        FROM events_counts
+        WHERE actor = '${id}'
+        GROUP BY actor;
+      `,
+      format: 'JSONEachRow'
+    });
+
+    const result = await rows.json<
+      Array<{
+        actor: string;
+        most_common_country: string;
+        most_common_region: string;
+        most_common_city: string;
+        number_of_events: string;
+        most_common_os: string;
+        most_common_browser: string;
+        most_common_browser_version: string;
+      }>
+    >();
+
+    return response({
+      success: true,
+      result: result[0]
+        ? {
+            actor: result[0].actor,
+            country: result[0].most_common_country,
+            region: result[0].most_common_region,
+            city: result[0].most_common_city,
+            events: parseInt(result[0].number_of_events),
+            os: result[0].most_common_os,
+            browser: result[0].most_common_browser,
+            version: result[0].most_common_browser_version
+          }
+        : null
+    });
+  } catch (error) {
+    throw error;
+  }
+};

packages/workers/stats/src/index.ts

@@ -3,6 +3,7 @@ import response from '@hey/lib/response';
 import { createCors, error, Router, status } from 'itty-router';
 
 import haveUsedHey from './handlers/haveUsedHey';
+import profileDetails from './handlers/profileDetails';
 import profileImpressions from './handlers/profileImpressions';
 import publicationViews from './handlers/publicationViews';
 import streaksCalendar from './handlers/streaksCalendar';
@@ -28,6 +29,7 @@ router
   )
   .post('/publicationViews', publicationViews)
   .get('/profileImpressions', profileImpressions)
+  .get('/profileDetails', profileDetails)
   .get('/haveUsedHey/:id', haveUsedHey)
   .get('/streaks/:id', streaksCalendar)
   .get('/streaks/:id/:date', streaksList)