feat: migrate feeds worker to next api (#4021)

heyxyz · Nov 17, 2023 · ab81314 · ab81314 · vercel · Nov 17, 2023
1 parent 3bd1169
commit ab81314
Show file tree

Hide file tree

Showing 30 changed files with 62 additions and 284 deletions.
diff --git a/apps/api/pages/api/feed/getPublicationIds.ts b/apps/api/pages/api/feed/getPublicationIds.ts
@@ -0,0 +1,45 @@
+import { AlgorithmProvider } from '@hey/data/enums';
+import { Errors } from '@hey/data/errors';
+import type { NextApiRequest, NextApiResponse } from 'next';
+import allowCors from 'utils/allowCors';
+import { CACHE_AGE } from 'utils/constants';
+import heyFeed from 'utils/feeds/providers/hey/heyFeed';
+import k3lFeed from 'utils/feeds/providers/k3l/k3lFeed';
+
+const handler = async (req: NextApiRequest, res: NextApiResponse) => {
+  const provider = req.query.provider as string;
+  const strategy = req.query.strategy as string;
+  const profile = req.query.profile as string;
+  const limit = (parseInt(req.query?.limit as string) || 50) as number;
+  const offset = (parseInt(req.query?.offset as string) || 0) as number;
+
+  if (!provider || !strategy) {
+    return res.status(400).json({ success: false, error: Errors.NoBody });
+  }
+
+  try {
+    let ids: string[] = [];
+    switch (provider) {
+      case AlgorithmProvider.K3L:
+        ids = await k3lFeed(strategy, profile, limit, offset);
+        break;
+      case AlgorithmProvider.HEY:
+        ids = await heyFeed(strategy, limit, offset);
+        break;
+      default:
+        return res
+          .status(200)
+          .setHeader('Cache-Control', CACHE_AGE)
+          .json({ success: false, message: 'Invalid provider' });
+    }
+
+    return res
+      .status(200)
+      .setHeader('Cache-Control', CACHE_AGE)
+      .json({ success: true, ids });
+  } catch (error) {
+    throw error;
+  }
+};
+
+export default allowCors(handler);
diff --git a/...iders/hey/algorithms/heyMostInteracted.ts → ...iders/hey/algorithms/heyMostInteracted.ts b/...iders/hey/algorithms/heyMostInteracted.ts → ...iders/hey/algorithms/heyMostInteracted.ts
@@ -1,9 +1,7 @@
-import createClickhouseClient from '@hey/clickhouse/createClickhouseClient';
 import { Errors } from '@hey/data/errors';
 import { PUBLICATION } from '@hey/data/tracking';
-
-import randomizeIds from '../../../helpers/randomizeIds';
-import type { Env } from '../../../types';
+import createClickhouseClient from 'utils/createClickhouseClient';
+import randomizeIds from 'utils/feeds/randomizeIds';
 
 const interactionAndWeights = {
   [PUBLICATION.COLLECT_MODULE.COLLECT]: 10,
@@ -30,15 +28,14 @@ const generateWeightedCaseStatement = () => {
 
 const heyMostInteracted = async (
   limit: number,
-  offset: number,
-  env: Env
+  offset: number
 ): Promise<any[]> => {
   if (limit > 500) {
     throw new Error(Errors.Limit500);
   }
 
   try {
-    const client = createClickhouseClient(env.CLICKHOUSE_PASSWORD);
+    const client = createClickhouseClient();
     const rows = await client.query({
       query: `
         SELECT

diff --git a/...providers/hey/algorithms/heyMostViewed.ts → ...providers/hey/algorithms/heyMostViewed.ts b/...providers/hey/algorithms/heyMostViewed.ts → ...providers/hey/algorithms/heyMostViewed.ts
@@ -1,22 +1,16 @@
-import createClickhouseClient from '@hey/clickhouse/createClickhouseClient';
 import { Errors } from '@hey/data/errors';
 import { PAGEVIEW } from '@hey/data/tracking';
+import createClickhouseClient from 'utils/createClickhouseClient';
+import randomizeIds from 'utils/feeds/randomizeIds';
+import removeParamsFromString from 'utils/feeds/removeParamsFromString';
 
-import randomizeIds from '../../../helpers/randomizeIds';
-import removeParamsFromString from '../../../helpers/removeParamsFromString';
-import type { Env } from '../../../types';
-
-const heyMostViewed = async (
-  limit: number,
-  offset: number,
-  env: Env
-): Promise<any[]> => {
+const heyMostViewed = async (limit: number, offset: number): Promise<any[]> => {
   if (limit > 500) {
     throw new Error(Errors.Limit500);
   }
 
   try {
-    const client = createClickhouseClient(env.CLICKHOUSE_PASSWORD);
+    const client = createClickhouseClient();
     const rows = await client.query({
       query: `
         SELECT

diff --git a/...orkers/feeds/src/providers/hey/heyFeed.ts → .../api/utils/feeds/providers/hey/heyFeed.ts b/...orkers/feeds/src/providers/hey/heyFeed.ts → .../api/utils/feeds/providers/hey/heyFeed.ts
@@ -1,18 +1,12 @@
-import type { Env } from '../../types';
 import heyMostInteracted from './algorithms/heyMostInteracted';
 import heyMostViewed from './algorithms/heyMostViewed';
 
-const heyFeed = async (
-  strategy: string,
-  limit: number,
-  offset: number,
-  env: Env
-) => {
+const heyFeed = async (strategy: string, limit: number, offset: number) => {
   switch (strategy) {
     case 'mostviewed':
-      return await heyMostViewed(limit, offset, env);
+      return await heyMostViewed(limit, offset);
     case 'mostinteracted':
-      return await heyMostInteracted(limit, offset, env);
+      return await heyMostInteracted(limit, offset);
     default:
       return [];
   }

diff --git a/...providers/k3l/algorithms/k3lGlobalFeed.ts → ...providers/k3l/algorithms/k3lGlobalFeed.ts b/...providers/k3l/algorithms/k3lGlobalFeed.ts → ...providers/k3l/algorithms/k3lGlobalFeed.ts
@@ -1,6 +1,5 @@
 import urlcat from 'urlcat';
-
-import randomizeIds from '../../../helpers/randomizeIds';
+import randomizeIds from 'utils/feeds/randomizeIds';
 
 const k3lGlobalFeed = async (
   strategy: string,

diff --git a/...oviders/k3l/algorithms/k3lPersonalFeed.ts → ...oviders/k3l/algorithms/k3lPersonalFeed.ts b/...oviders/k3l/algorithms/k3lPersonalFeed.ts → ...oviders/k3l/algorithms/k3lPersonalFeed.ts
@@ -1,6 +1,5 @@
 import urlcat from 'urlcat';
-
-import randomizeIds from '../../../helpers/randomizeIds';
+import randomizeIds from 'utils/feeds/randomizeIds';
 
 const k3lPersonalFeed = async (
   strategy: string,

diff --git a/...orkers/feeds/src/providers/k3l/k3lFeed.ts → .../api/utils/feeds/providers/k3l/k3lFeed.ts b/...orkers/feeds/src/providers/k3l/k3lFeed.ts → .../api/utils/feeds/providers/k3l/k3lFeed.ts
diff --git a/...workers/feeds/src/helpers/randomizeIds.ts → apps/api/utils/feeds/randomizeIds.ts b/...workers/feeds/src/helpers/randomizeIds.ts → apps/api/utils/feeds/randomizeIds.ts
diff --git a/...eds/src/helpers/removeParamsFromString.ts → ...api/utils/feeds/removeParamsFromString.ts b/...eds/src/helpers/removeParamsFromString.ts → ...api/utils/feeds/removeParamsFromString.ts
diff --git a/docs/ports.md b/docs/ports.md
@@ -10,7 +10,6 @@ The following ports are exposed by the repository
 | 8084 | Snapshot relay worker | http://localhost:8084 | https://snapshot-relay.hey.xyz |
 | 8086 | Oembed worker         | http://localhost:8086 | https://oembed.hey.xyz         |
 | 8088 | Stats worker          | http://localhost:8088 | https://stats.hey.xyz          |
-| 8089 | Feeds gateway         | http://localhost:8089 | https://feeds.hey.xyz          |
 | 8090 | Preferences worker    | http://localhost:8090 | https://preferences.hey.xyz    |
 | 8091 | Groups worker         | http://localhost:8091 | https://groups.hey.xyz         |
 | 8092 | NFT worker            | http://localhost:8092 | https://nft.hey.xyz            |

diff --git a/package.json b/package.json
@@ -16,7 +16,7 @@
     "prettier:fix": "turbo run prettier:fix --parallel",
     "start": "turbo run start --parallel",
     "test:dev": "turbo run test:dev",
-    "test:e2e": "start-server-and-test start '8083|8086|8089' test:dev",
+    "test:e2e": "start-server-and-test start '8083|8086' test:dev",
     "typecheck": "turbo run typecheck --parallel",
     "typecheck:affected": "turbo run typecheck --parallel --filter=[origin/main]",
     "workers:deploy": "turbo run worker:deploy --parallel"

diff --git a/packages/data/constants.ts b/packages/data/constants.ts
@@ -71,9 +71,6 @@ export const OEMBED_WORKER_URL = IS_PRODUCTION
 export const STATS_WORKER_URL = IS_PRODUCTION
   ? 'https://stats.hey.xyz'
   : 'http://localhost:8088';
-export const FEEDS_WORKER_URL = IS_PRODUCTION
-  ? 'https://feeds.hey.xyz'
-  : 'http://localhost:8089';
 export const PREFERENCES_WORKER_URL = IS_PRODUCTION
   ? 'https://preferences.hey.xyz'
   : 'http://localhost:8090';

diff --git a/packages/lib/getPublicationIds.ts b/packages/lib/getPublicationIds.ts
@@ -1,4 +1,4 @@
-import { FEEDS_WORKER_URL } from '@hey/data/constants';
+import { HEY_API_URL } from '@hey/data/constants';
 import axios from 'axios';
 
 /**
@@ -16,7 +16,7 @@ const getPublicationIds = async (
   profile?: string
 ) => {
   try {
-    const response = await axios.get(`${FEEDS_WORKER_URL}/ids`, {
+    const response = await axios.get(`${HEY_API_URL}/feed/getPublicationIds`, {
       params: {
         provider,
         strategy,

diff --git a/packages/workers/feeds/.dev.vars.example b/packages/workers/feeds/.dev.vars.example
diff --git a/packages/workers/feeds/.eslintrc.js b/packages/workers/feeds/.eslintrc.js
diff --git a/packages/workers/feeds/README.md b/packages/workers/feeds/README.md
diff --git a/packages/workers/feeds/package.json b/packages/workers/feeds/package.json
diff --git a/packages/workers/feeds/src/constants.ts b/packages/workers/feeds/src/constants.ts
diff --git a/packages/workers/feeds/src/handlers/getPublicationIds.spec.ts b/packages/workers/feeds/src/handlers/getPublicationIds.spec.ts
diff --git a/packages/workers/feeds/src/handlers/getPublicationIds.ts b/packages/workers/feeds/src/handlers/getPublicationIds.ts
diff --git a/packages/workers/feeds/src/helpers/buildRequest.ts b/packages/workers/feeds/src/helpers/buildRequest.ts
diff --git a/packages/workers/feeds/src/index.ts b/packages/workers/feeds/src/index.ts
diff --git a/packages/workers/feeds/src/providers/hey/README.md b/packages/workers/feeds/src/providers/hey/README.md