feat: clickhouse client

apps/api/.env.example

@@ -1,7 +1,8 @@
 EVER_ACCESS_KEY=
 EVER_ACCESS_SECRET=
 LOGTAIL_API_KEY=
-CH_INGEST_ENDPOINT=
+CLICKHOUSE_URL=
+CLICKHOUSE_PASSWORD=
 IP_API_KEY=
 WALLET_PRIVATE_KEY= #pk without 0x
 LIVEPEER_API_TOKEN=

apps/api/package.json

@@ -15,6 +15,7 @@
   },
   "dependencies": {
     "@aws-sdk/client-sts": "3.614.0",
+    "@clickhouse/client": "^1.4.0",
     "@hono/node-server": "^1.11.2",
     "@hono/zod-validator": "^0.2.2",
     "@lens-protocol/metadata": "^1.2.0",
@@ -30,10 +31,10 @@
   "devDependencies": {
     "@tape.xyz/config": "workspace:*",
     "@tape.xyz/generic": "workspace:*",
+    "@types/node": "^20.11.17",
     "@types/ua-parser-js": "^0.7.39",
     "prettier": "^3.3.2",
     "tsx": "^4.7.1",
-    "@types/node": "^20.11.17",
     "typescript": "5.5.3"
   }
 }

apps/api/src/helpers/tower/clickhouse.ts

@@ -0,0 +1,11 @@
+import { createClient } from '@clickhouse/client'
+
+const clickhouseClient = createClient({
+  compression: { request: true, response: true },
+  keep_alive: { enabled: true },
+  username: 'clickhouse',
+  password: process.env.CLICKHOUSE_PASSWORD,
+  url: process.env.CLICKHOUSE_URL
+})
+
+export default clickhouseClient

apps/api/src/routes/tower.ts

@@ -7,6 +7,7 @@ import { any, object, string } from 'zod'
 
 import { ERROR_MESSAGE } from '@/helpers/constants'
 import checkEventExistence from '@/helpers/tower/checkEventExistence'
+import clickhouseClient from '@/helpers/tower/clickhouse'
 
 const app = new Hono()
 
@@ -46,7 +47,6 @@ app.post('/', zValidator('json', validationSchema), async (c) => {
     } | null = null
 
     const IP_API_KEY = process.env.IP_API_KEY!
-    const CH_INGEST_ENDPOINT = process.env.CH_INGEST_ENDPOINT!
     try {
       const ipResponse = await fetch(
         `https://pro.ip-api.com/json/${ip}?key=${IP_API_KEY}`
@@ -62,58 +62,33 @@ app.post('/', zValidator('json', validationSchema), async (c) => {
     const utmTerm = parsedUrl.searchParams.get('utm_term') || null
     const utmContent = parsedUrl.searchParams.get('utm_content') || null
 
-    const body = `
-        INSERT INTO events (
-          name,
-          actor,
-          properties,
-          url,
-          city,
-          country,
-          region,
-          referrer,
-          platform,
-          browser,
-          browser_version,
-          os,
-          utm_source,
-          utm_medium,
-          utm_campaign,
-          utm_term,
-          utm_content,
-          fingerprint
-        ) VALUES (
-          '${name}',
-          ${actor ? `'${actor}'` : null},
-          ${properties ? `'${JSON.stringify(properties)}'` : null},
-          ${url ? `'${url}'` : null},
-          ${ipData?.city ? `'${ipData?.city}'` : null},
-          ${ipData?.country ? `'${ipData?.country}'` : null},
-          ${ipData?.regionName ? `'${ipData?.regionName}'` : null},
-          ${referrer ? `'${referrer}'` : null},
-          ${platform ? `'${platform}'` : null},
-          ${ua.browser.name ? `'${ua.browser.name}'` : null},
-          ${ua.browser.version ? `'${ua.os.version}'` : null},
-          ${ua.os.name ? `'${ua.os.name}'` : null},
-          ${utmSource ? `'${utmSource}'` : null},
-          ${utmMedium ? `'${utmMedium}'` : null},
-          ${utmCampaign ? `'${utmCampaign}'` : null},
-          ${utmTerm ? `'${utmTerm}'` : null},
-          ${utmContent ? `'${utmContent}'` : null},
-          ${fingerprint ? `'${fingerprint}'` : null}
-        )
-      `
+    const value = {
+      name,
+      actor: actor || null,
+      properties: properties || null,
+      url: url || null,
+      city: ipData?.city || null,
+      country: ipData?.country || null,
+      region: ipData?.regionName || null,
+      referrer: referrer || null,
+      platform: platform || null,
+      browser: ua.browser.name || null,
+      browser_version: ua.browser.version || null,
+      os: ua.os.name || null,
+      utm_source: utmSource || null,
+      utm_medium: utmMedium || null,
+      utm_campaign: utmCampaign || null,
+      utm_term: utmTerm || null,
+      utm_content: utmContent || null,
+      fingerprint: fingerprint || null
+    }
 
-    const clickhouseResponse = await fetch(CH_INGEST_ENDPOINT, {
-      method: 'POST',
-      headers: { 'Content-Type': 'application/json' },
-      body
+    await clickhouseClient.insert({
+      format: 'JSONEachRow',
+      table: 'events',
+      values: [value]
     })
 
-    if (clickhouseResponse.status !== 200) {
-      return c.json({ success: false, message: ERROR_MESSAGE })
-    }
-
     return c.json({ success: true })
   } catch {
     return c.json({ success: false, message: ERROR_MESSAGE })

package.json

@@ -5,12 +5,12 @@
   "private": true,
   "scripts": {
     "preinstall": "npx only-allow pnpm",
-    "build": "pnpm -r run build",
-    "dev": "pnpm -r run dev",
-    "start": "pnpm -r run start",
-    "codegen": "pnpm -r run codegen",
-    "lint": "pnpm -r run lint",
-    "typecheck": "pnpm -r run typecheck",
+    "build": "pnpm -r --parallel run build",
+    "dev": "pnpm -r --parallel run dev",
+    "start": "pnpm -r --parallel run start",
+    "codegen": "pnpm -r --parallel run codegen",
+    "lint": "pnpm -r --parallel run lint",
+    "typecheck": "pnpm -r --parallel run typecheck",
     "prepare": "husky",
     "format": "prettier --write \"**/*.{ts,tsx,md}\""
   },