feat: add impressions worker (#3972)

heyxyz · Nov 8, 2023 · 71473bb · 71473bb · vercel · Nov 8, 2023
2 parents 51d11b3 + ac792d4
commit 71473bb
Show file tree

Hide file tree

Showing 25 changed files with 255 additions and 956 deletions.
diff --git a/apps/web/public/impressions.js b/apps/web/public/impressions.js
@@ -1,33 +1,27 @@
-const postsEndpoint = 'https://ens.hey.xyz';
-const postsVisibilityInterval = 5000;
-let visiblePostsSet = new Set();
+const impressionsEndpoint = 'https://impressions.hey.xyz/ingest';
+const publicationsVisibilityInterval = 5000;
+let visiblePublicationsSet = new Set();
 
-function sendVisiblePostsToServer() {
-  const postsToSend = Array.from(visiblePostsSet);
+function sendVisiblePublicationsToServer() {
+  const publicationsToSend = Array.from(visiblePublicationsSet);
 
-  if (postsToSend.length > 0) {
-    visiblePostsSet.clear();
-    fetch(postsEndpoint, {
+  if (publicationsToSend.length > 0) {
+    visiblePublicationsSet.clear();
+    fetch(impressionsEndpoint, {
       method: 'POST',
       headers: { 'Content-Type': 'application/json' },
-      body: JSON.stringify({ visiblePosts: postsToSend }),
+      body: JSON.stringify({ ids: publicationsToSend }),
       keepalive: true
     })
-      .then((response) => {
-        if (!response.ok) {
-          console.error('Failed to send visible posts:', response);
-        }
-      })
-      .catch((error) => {
-        console.error('Error sending visible posts:', error);
-      });
+      .then(() => {})
+      .catch(() => {});
   }
 }
 
-setInterval(sendVisiblePostsToServer, postsVisibilityInterval);
+setInterval(sendVisiblePublicationsToServer, publicationsVisibilityInterval);
 
 self.addEventListener('message', (event) => {
-  if (event.data && event.data.type === 'POST_VISIBLE') {
-    visiblePostsSet.add(event.data.postId);
+  if (event.data && event.data.type === 'PUBLICATION_VISIBLE') {
+    visiblePublicationsSet.add(event.data.id);
   }
 });
diff --git a/apps/web/src/components/Common/Providers/LeafwatchProvider.tsx b/apps/web/src/components/Common/Providers/LeafwatchProvider.tsx
@@ -1,51 +1,18 @@
-import { IMPRESSIONS_SERVICE_URL, IS_MAINNET } from '@hey/data/constants';
 import { type FC } from 'react';
-import useWebSocket from 'react-use-websocket';
 import { useLeafwatchPersistStore } from 'src/store/useLeafwatchPersistStore';
-import { useLeafwatchStore } from 'src/store/useLeafwatchStore';
-import { useEffectOnce, useInterval, useUpdateEffect } from 'usehooks-ts';
+import { useEffectOnce } from 'usehooks-ts';
 import { v4 as uuid } from 'uuid';
 
 const LeafwatchProvider: FC = () => {
   const viewerId = useLeafwatchPersistStore((state) => state.viewerId);
   const setViewerId = useLeafwatchPersistStore((state) => state.setViewerId);
-  const viewedPublication = useLeafwatchStore(
-    (state) => state.viewedPublication
-  );
-
-  const { sendJsonMessage, readyState } = useWebSocket(
-    IMPRESSIONS_SERVICE_URL,
-    {
-      reconnectAttempts: 5,
-      reconnectInterval: 5000,
-      shouldReconnect: () => true
-    }
-  );
 
   useEffectOnce(() => {
-    sendJsonMessage({ type: 'connection_init' });
     if (!viewerId) {
       setViewerId(uuid());
     }
   });
 
-  useInterval(() => {
-    sendJsonMessage({ type: 'connection_init' });
-  }, 10000);
-
-  useUpdateEffect(() => {
-    if (readyState === 1 && viewedPublication && IS_MAINNET) {
-      sendJsonMessage({
-        id: '1',
-        type: 'start',
-        payload: JSON.stringify({
-          viewer_id: viewerId,
-          publication_id: viewedPublication
-        })
-      });
-    }
-  }, [readyState, viewedPublication]);
-
   return null;
 };
 

diff --git a/apps/web/src/components/Notification/Type/ActedNotification.tsx b/apps/web/src/components/Notification/Type/ActedNotification.tsx
@@ -6,7 +6,6 @@ import { isMirrorPublication } from '@hey/lib/publicationHelpers';
 import Link from 'next/link';
 import plur from 'plur';
 import type { FC } from 'react';
-import { useLeafwatchStore } from 'src/store/useLeafwatchStore';
 import { useEffectOnce } from 'usehooks-ts';
 
 import AggregatedNotificationTitle from '../AggregatedNotificationTitle';
@@ -17,10 +16,6 @@ interface ActedNotificationProps {
 }
 
 const ActedNotification: FC<ActedNotificationProps> = ({ notification }) => {
-  const setViewedPublication = useLeafwatchStore(
-    (state) => state.setViewedPublication
-  );
-
   const publication = notification?.publication;
   const targetPublication = isMirrorPublication(publication)
     ? publication.mirrorOn
@@ -38,8 +33,11 @@ const ActedNotification: FC<ActedNotificationProps> = ({ notification }) => {
   const type = notification?.publication.__typename;
 
   useEffectOnce(() => {
-    if (notification?.publication) {
-      setViewedPublication(notification?.publication.id);
+    if (notification?.publication.id && navigator.serviceWorker.controller) {
+      navigator.serviceWorker.controller.postMessage({
+        type: 'PUBLICATION_VISIBLE',
+        id: notification.publication.id
+      });
     }
   });
 

diff --git a/apps/web/src/components/Notification/Type/MirrorNotification.tsx b/apps/web/src/components/Notification/Type/MirrorNotification.tsx
@@ -5,7 +5,6 @@ import getPublicationData from '@hey/lib/getPublicationData';
 import Link from 'next/link';
 import plur from 'plur';
 import type { FC } from 'react';
-import { useLeafwatchStore } from 'src/store/useLeafwatchStore';
 import { useEffectOnce } from 'usehooks-ts';
 
 import AggregatedNotificationTitle from '../AggregatedNotificationTitle';
@@ -16,10 +15,6 @@ interface MirrorNotificationProps {
 }
 
 const MirrorNotification: FC<MirrorNotificationProps> = ({ notification }) => {
-  const setViewedPublication = useLeafwatchStore(
-    (state) => state.setViewedPublication
-  );
-
   const metadata = notification?.publication.metadata;
   const filteredContent = getPublicationData(metadata)?.content || '';
   const mirrors = notification?.mirrors;
@@ -33,8 +28,11 @@ const MirrorNotification: FC<MirrorNotificationProps> = ({ notification }) => {
   const type = notification?.publication.__typename;
 
   useEffectOnce(() => {
-    if (notification?.publication) {
-      setViewedPublication(notification?.publication.id);
+    if (notification?.publication.id && navigator.serviceWorker.controller) {
+      navigator.serviceWorker.controller.postMessage({
+        type: 'PUBLICATION_VISIBLE',
+        id: notification.publication.id
+      });
     }
   });
 

diff --git a/apps/web/src/components/Notification/Type/QuoteNotification.tsx b/apps/web/src/components/Notification/Type/QuoteNotification.tsx
@@ -4,7 +4,6 @@ import { QuoteNotification } from '@hey/lens';
 import getPublicationData from '@hey/lib/getPublicationData';
 import Link from 'next/link';
 import type { FC } from 'react';
-import { useLeafwatchStore } from 'src/store/useLeafwatchStore';
 import { useEffectOnce } from 'usehooks-ts';
 
 import AggregatedNotificationTitle from '../AggregatedNotificationTitle';
@@ -15,10 +14,6 @@ interface QuoteNotificationProps {
 }
 
 const QuoteNotification: FC<QuoteNotificationProps> = ({ notification }) => {
-  const setViewedPublication = useLeafwatchStore(
-    (state) => state.setViewedPublication
-  );
-
   const metadata = notification?.quote.metadata;
   const filteredContent = getPublicationData(metadata)?.content || '';
   const firstProfile = notification.quote.by;
@@ -27,8 +22,11 @@ const QuoteNotification: FC<QuoteNotificationProps> = ({ notification }) => {
   const type = notification.quote.quoteOn.__typename;
 
   useEffectOnce(() => {
-    if (notification?.quote) {
-      setViewedPublication(notification?.quote.id);
+    if (notification?.quote.id && navigator.serviceWorker.controller) {
+      navigator.serviceWorker.controller.postMessage({
+        type: 'PUBLICATION_VISIBLE',
+        id: notification.quote.id
+      });
     }
   });
 

diff --git a/apps/web/src/components/Notification/Type/ReactionNotification.tsx b/apps/web/src/components/Notification/Type/ReactionNotification.tsx
@@ -5,7 +5,6 @@ import getPublicationData from '@hey/lib/getPublicationData';
 import Link from 'next/link';
 import plur from 'plur';
 import type { FC } from 'react';
-import { useLeafwatchStore } from 'src/store/useLeafwatchStore';
 import { useEffectOnce } from 'usehooks-ts';
 
 import AggregatedNotificationTitle from '../AggregatedNotificationTitle';
@@ -18,10 +17,6 @@ interface ReactionNotificationProps {
 const ReactionNotification: FC<ReactionNotificationProps> = ({
   notification
 }) => {
-  const setViewedPublication = useLeafwatchStore(
-    (state) => state.setViewedPublication
-  );
-
   const metadata = notification?.publication.metadata;
   const filteredContent = getPublicationData(metadata)?.content || '';
   const reactions = notification?.reactions;
@@ -35,8 +30,11 @@ const ReactionNotification: FC<ReactionNotificationProps> = ({
   const type = notification?.publication.__typename;
 
   useEffectOnce(() => {
-    if (notification?.publication) {
-      setViewedPublication(notification?.publication.id);
+    if (notification?.publication.id && navigator.serviceWorker.controller) {
+      navigator.serviceWorker.controller.postMessage({
+        type: 'PUBLICATION_VISIBLE',
+        id: notification.publication.id
+      });
     }
   });
 

diff --git a/apps/web/src/components/Publication/FullPublication.tsx b/apps/web/src/components/Publication/FullPublication.tsx
@@ -3,7 +3,6 @@ import getAppName from '@hey/lib/getAppName';
 import { isMirrorPublication } from '@hey/lib/publicationHelpers';
 import { formatDate } from '@lib/formatTime';
 import { type FC } from 'react';
-import { useLeafwatchStore } from 'src/store/useLeafwatchStore';
 import { useEffectOnce } from 'usehooks-ts';
 
 import PublicationActions from './Actions';
@@ -19,19 +18,18 @@ interface FullPublicationProps {
 }
 
 const FullPublication: FC<FullPublicationProps> = ({ publication }) => {
-  const setViewedPublication = useLeafwatchStore(
-    (state) => state.setViewedPublication
-  );
-
   const targetPublication = isMirrorPublication(publication)
     ? publication?.mirrorOn
     : publication;
 
   const { metadata, createdAt } = targetPublication;
 
   useEffectOnce(() => {
-    if (targetPublication.id) {
-      setViewedPublication(targetPublication.id);
+    if (targetPublication.id && navigator.serviceWorker.controller) {
+      navigator.serviceWorker.controller.postMessage({
+        type: 'PUBLICATION_VISIBLE',
+        id: targetPublication.id
+      });
     }
   });
 

diff --git a/apps/web/src/components/Publication/SinglePublication.tsx b/apps/web/src/components/Publication/SinglePublication.tsx
@@ -5,7 +5,6 @@ import { isMirrorPublication } from '@hey/lib/publicationHelpers';
 import cn from '@hey/ui/cn';
 import { type FC } from 'react';
 import { useInView } from 'react-cool-inview';
-import { useLeafwatchStore } from 'src/store/useLeafwatchStore';
 
 import PublicationActions from './Actions';
 import ModAction from './Actions/ModAction';
@@ -38,10 +37,6 @@ const SinglePublication: FC<SinglePublicationProps> = ({
   isFirst = false,
   isLast = false
 }) => {
-  const setViewedPublication = useLeafwatchStore(
-    (state) => state.setViewedPublication
-  );
-
   const firstComment = feedItem?.comments?.[0];
   const rootPublication = feedItem
     ? firstComment
@@ -60,8 +55,8 @@ const SinglePublication: FC<SinglePublicationProps> = ({
 
       if (rootPublication.id && navigator.serviceWorker.controller) {
         navigator.serviceWorker.controller.postMessage({
-          type: 'POST_VISIBLE',
-          postId: rootPublication.id
+          type: 'PUBLICATION_VISIBLE',
+          id: rootPublication.id
         });
       }
     }

diff --git a/apps/web/src/store/useLeafwatchStore.ts b/apps/web/src/store/useLeafwatchStore.ts
diff --git a/docs/ports.md b/docs/ports.md
@@ -6,7 +6,6 @@ The following ports are exposed by the repository
 | ---- | -------------------------- | --------------------- | ------------------------------ |
 | 4783 | Web app                    | http://localhost:4783 | https://hey.xyz                |
 | 4784 | Prerender app              | http://localhost:4784 | https://prerender.hey.xyz      |
-| 5001 | Impressions service        | http://localhost:5001 | https://impressions.hey.xyz    |
 | 8082 | STS token generator worker | http://localhost:8082 | https://sts.hey.xyz            |
 | 8083 | Metadata worker            | http://localhost:8083 | https://metadata.hey.xyz       |
 | 8084 | Snapshot relay worker      | http://localhost:8084 | https://snapshot-relay.hey.xyz |
@@ -20,3 +19,4 @@ The following ports are exposed by the repository
 | 8092 | NFT worker                 | http://localhost:8092 | https://nft.hey.xyz            |
 | 8093 | Staff Picks worker         | http://localhost:8093 | https://staff-picks.hey.xyz    |
 | 8094 | Live worker                | http://localhost:8094 | https://live.hey.xyz           |
+| 8094 | Impressions worker         | http://localhost:8094 | https://impressions.hey.xyz    |
diff --git a/packages/workers/impressions/.env.example → ...ges/workers/impressions/.dev.vars.example b/packages/workers/impressions/.env.example → ...ges/workers/impressions/.dev.vars.example
diff --git a/packages/workers/impressions/.eslintrc.js b/packages/workers/impressions/.eslintrc.js
@@ -2,6 +2,5 @@ module.exports = {
   extends: [require.resolve('@hey/config/eslint/base.js')],
   rules: {
     'import/no-anonymous-default-export': 'off'
-  },
-  ignorePatterns: ['**/dist/**/*']
+  }
 };
diff --git a/packages/workers/impressions/.gitignore b/packages/workers/impressions/.gitignore
diff --git a/packages/workers/impressions/README.md b/packages/workers/impressions/README.md
@@ -0,0 +1,15 @@
+# Impressions worker
+
+## Clickhouse Schema
+
+```sql
+CREATE TABLE impressions (
+  id UUID,
+  viewer_id String,
+  publication_id String,
+  viewed_at DateTime64(3, 'UTC') DEFAULT now64(3)
+) ENGINE = MergeTree()
+PARTITION BY toYYYYMM(viewed_at)
+ORDER BY (viewer_id, publication_id, viewed_at)
+SETTINGS index_granularity = 8192;
+```