(feature) Local time

apps/production/src/analytics/analytics.controller.ts

@@ -123,6 +123,7 @@ const analyticsDTO = (
   cc: string,
   rg: string,
   ct: string,
+  lt: number | string,
   keys: string[],
   values: string[],
   sdur: number | string,
@@ -145,6 +146,7 @@ const analyticsDTO = (
     cc,
     rg,
     ct,
+    lt,
     keys,
     values,
     sdur,
@@ -206,6 +208,7 @@ const customLogDTO = (
   cc: string,
   rg: string,
   ct: string,
+  lt: number | string,
   keys: string[],
   values: string[],
 ): Array<string | number | string[]> => {
@@ -225,6 +228,7 @@ const customLogDTO = (
     cc,
     rg,
     ct,
+    lt,
     keys,
     values,
     dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),
@@ -1337,13 +1341,17 @@ export class AnalyticsController {
       city = 'NULL',
       region = 'NULL',
       country = 'NULL',
+      localTime: localTimeUnprepared,
     } = getGeoDetails(ip, eventsDTO.tz)
 
     const ua = UAParser(userAgent)
     const dv = ua.device.type || 'desktop'
     const br = ua.browser.name
     const os = ua.os.name
 
+    const localTime =
+      localTimeUnprepared === null ? 'NULL' : localTimeUnprepared
+
     const sessionHash = getSessionKey(ip, userAgent, eventsDTO.pid, salt)
     const [, psid] = await this.analyticsService.isUnique(sessionHash)
 
@@ -1363,6 +1371,7 @@ export class AnalyticsController {
       country,
       region,
       city,
+      localTime,
       _keys(eventsDTO.meta),
       _values(eventsDTO.meta),
     )
@@ -1438,12 +1447,16 @@ export class AnalyticsController {
       city = 'NULL',
       region = 'NULL',
       country = 'NULL',
+      localTime: localTimeUnprepared,
     } = getGeoDetails(ip, logDTO.tz)
     const ua = UAParser(userAgent)
     const dv = ua.device.type || 'desktop'
     const br = ua.browser.name
     const os = ua.os.name
 
+    const localTime =
+      localTimeUnprepared === null ? 'NULL' : localTimeUnprepared
+
     const dto = analyticsDTO(
       psid,
       sessionHash,
@@ -1461,6 +1474,7 @@ export class AnalyticsController {
       country,
       region,
       city,
+      localTime,
       _keys(logDTO.meta),
       _values(logDTO.meta),
       0,
@@ -1554,12 +1568,16 @@ export class AnalyticsController {
       city = 'NULL',
       region = 'NULL',
       country = 'NULL',
+      localTime: localTimeUnprepared,
     } = getGeoDetails(ip)
     const ua = UAParser(userAgent)
     const dv = ua.device.type || 'desktop'
     const br = ua.browser.name
     const os = ua.os.name
 
+    const localTime =
+      localTimeUnprepared === null ? 'NULL' : localTimeUnprepared
+
     const dto = analyticsDTO(
       psid,
       sessionHash,
@@ -1577,6 +1595,7 @@ export class AnalyticsController {
       country,
       region,
       city,
+      localTime,
       [],
       [],
       0,

apps/production/src/common/utils.ts

@@ -1,8 +1,11 @@
 import * as path from 'path'
 import * as fs from 'fs'
+// import * as tzLookup from '@photostructure/tz-lookup'
 import { Reader, CityResponse } from 'maxmind'
 import { HttpException } from '@nestjs/common'
 import { hash } from 'blake3'
+import * as dayjs from 'dayjs'
+import * as dayjsTimezone from 'dayjs/plugin/timezone'
 import timezones from 'countries-and-timezones'
 import * as randomstring from 'randomstring'
 import * as _sample from 'lodash/sample'
@@ -13,8 +16,12 @@
 import * as _round from 'lodash/round'
 import * as _split from 'lodash/split'
 
+const tzLookup = require('@photostructure/tz-lookup')
+
 import { redis, isDevelopment, isProxiedByCloudflare } from './constants'
 
+dayjs.extend(dayjsTimezone)
+
 const marketingTips = {
   en: [
     'Blog frequently and consistently to build search engine presence.\nTo get website visitors, you need lots and lots of content. Your prospects are using Google every day to search for what they want. One of the most reliable ways to show up in Google results is to blog about lots of topics your customers care about.',
@@ -188,6 +195,27 @@
   country?: string
   region?: string
   city?: string
+  localTime?: number
+}
+
+const getLocalTime = (latitude: number, longitude: number): number | null => {
+  if (!latitude || !longitude) {
+    return null
+  }
+
+  try {
+    const localTz = tzLookup(latitude, longitude)
+
+    if (!localTz) {
+      return null
+    }
+
+    return dayjs().tz(localTz).hour()
+  } catch {
+    //
+  }
+
+  return null
 }
 
 const getGeoDetails = (ip: string, tz?: string): IPGeoDetails => {
@@ -199,11 +227,15 @@
   const city = data?.city?.names?.en
   const region = data?.subdivisions?.[0]?.names?.en
 
+  // TODO: Buy db-ip IP to Location database to avoid manually looking up the local time
   if (country) {
+    const { latitude, longitude } = data?.location || {}
+
     return {
       country,
       city,
       region,
+      localTime: getLocalTime(latitude, longitude),
     }
   }
 
@@ -214,6 +246,7 @@
     country: tzCountry,
     city: null,
     region: null,
+    localTime: null,
   }
 }
 

migrations/clickhouse/2024_04_28.js

@@ -0,0 +1,77 @@
+const { queriesRunner, dbName } = require('./setup')
+
+const queries = [
+  // Analytics table: added lt (local time) column
+  `DROP TABLE IF EXISTS ${dbName}.analytics_temp`,
+  `CREATE TABLE IF NOT EXISTS ${dbName}.analytics_temp
+  (
+    psid Nullable(UInt64),
+    sid Nullable(String),
+    pid FixedString(12),
+    pg Nullable(String),
+    prev Nullable(String),
+    dv LowCardinality(Nullable(String)),
+    br LowCardinality(Nullable(String)),
+    os LowCardinality(Nullable(String)),
+    lc LowCardinality(Nullable(String)),
+    ref Nullable(String),
+    so Nullable(String),
+    me Nullable(String),
+    ca Nullable(String),
+    cc Nullable(FixedString(2)),
+    rg LowCardinality(Nullable(String)),
+    ct Nullable(String),
+    lt Nullable(UInt8),
+    sdur Nullable(UInt32), 
+    unique UInt8,
+    created DateTime('UTC')
+  )
+  ENGINE = MergeTree()
+  PARTITION BY toYYYYMM(created)
+  ORDER BY (pid, created);`,
+
+  `INSERT INTO ${dbName}.analytics_temp (psid, sid, pid, pg, prev, dv, br, os, lc, ref, so, me, ca, cc, rg, ct, lt, sdur, unique, created)
+  SELECT psid, sid, pid, pg, prev, dv, br, os, lc, ref, so, me, ca, cc, rg, ct, NULL, sdur, unique, created FROM ${dbName}.analytics`,
+
+  `DROP TABLE ${dbName}.analytics`,
+  `RENAME TABLE ${dbName}.analytics_temp TO ${dbName}.analytics`,
+
+  // Custom events table: added lt (local time) column
+  `DROP TABLE IF EXISTS ${dbName}.customEV_temp`,
+  `CREATE TABLE IF NOT EXISTS ${dbName}.customEV_temp
+  (
+    psid Nullable(UInt64),
+    pid FixedString(12),
+    ev String,
+    pg Nullable(String),
+    dv LowCardinality(Nullable(String)),
+    br LowCardinality(Nullable(String)),
+    os LowCardinality(Nullable(String)),
+    lc LowCardinality(Nullable(String)),
+    ref Nullable(String),
+    so Nullable(String),
+    me Nullable(String),
+    ca Nullable(String),
+    cc Nullable(FixedString(2)),
+    rg LowCardinality(Nullable(String)),
+    ct Nullable(String),
+    lt Nullable(UInt8),
+    meta Nested
+    (
+      key String,
+      value String
+    ),
+    created DateTime('UTC')
+  )
+  ENGINE = MergeTree()
+  PARTITION BY toYYYYMM(created)
+  ORDER BY (pid, created);`,
+
+  `INSERT INTO ${dbName}.customEV_temp (psid, pid, ev, pg, dv, br, os, lc, ref, so, me, ca, cc, rg, ct, lt, meta.key, meta.value, created)
+  SELECT psid, pid, ev, pg, dv, br, os, lc, ref, so, me, ca, cc, rg, ct, NULL, meta.key, meta.value, created FROM ${dbName}.customEV`,
+
+  `DROP TABLE ${dbName}.customEV`,
+  `RENAME TABLE ${dbName}.customEV_temp TO ${dbName}.customEV`,
+]
+
+queriesRunner(queries)

migrations/clickhouse/initialise_database.js

@@ -23,6 +23,7 @@ const CLICKHOUSE_INIT_QUERIES = [
     cc Nullable(FixedString(2)),
     rg LowCardinality(Nullable(String)),
     ct Nullable(String),
+    lt Nullable(UInt8),
     meta Nested
     (
       key String,
@@ -54,6 +55,7 @@ const CLICKHOUSE_INIT_QUERIES = [
     cc Nullable(FixedString(2)),
     rg LowCardinality(Nullable(String)),
     ct Nullable(String),
+    lt Nullable(UInt8),
     meta Nested
     (
       key String,

package.json

@@ -41,6 +41,7 @@
     "@nestjs/terminus": "^10.2.3",
     "@nestjs/typeorm": "^10.0.1",
     "@paypal/payouts-sdk": "^1.1.1",
+    "@photostructure/tz-lookup": "^10.0.0",
     "@sentry/node": "^8.0.0-rc.3",
     "axios": "^1.6.8",
     "bcrypt": "^5.1.1",