feat: fix various bugs & try out better message encryption & priv API

.env.example

@@ -7,4 +7,5 @@ IMGUR_CLIENT_ID="imgur client id" # get it from imgur, needed to parse imgur url
 VOTE_WEBHOOK_URL="your vote webhook url" # for sending that someone voted for the bot (optional)
 NETWORK_API_KEY="your network api key" # for posting to global chat (ask devoid)
 NODE_ENV=development # change to production when deploying
-DEBUG=false # set to true to enable debug logging
+DEBUG=false # set to true to enable debug logging
+PORT=3000 # or anything else for production

package.json

@@ -21,7 +21,7 @@
   "type": "module",
   "dependencies": {
     "@prisma/client": "^5.15.0",
-    "@sentry/node": "^8.8.0",
+    "@sentry/node": "^8.9.2",
     "@tensorflow/tfjs-node": "^4.20.0",
     "@top-gg/sdk": "^3.1.6",
     "common-tags": "^1.8.2",
@@ -40,7 +40,7 @@
     "winston": "^3.13.0"
   },
   "devDependencies": {
-    "@stylistic/eslint-plugin": "^2.1.0",
+    "@stylistic/eslint-plugin": "^2.2.1",
     "@types/common-tags": "^1.8.4",
     "@types/express": "^4.17.21",
     "@types/js-yaml": "^4.0.9",
@@ -49,13 +49,13 @@
     "@types/source-map-support": "^0.5.10",
     "cz-conventional-changelog": "^3.3.0",
     "eslint": "8.57.0",
-    "lint-staged": "^15.2.5",
-    "prettier": "^3.3.1",
+    "lint-staged": "^15.2.7",
+    "prettier": "^3.3.2",
     "prisma": "^5.15.0",
     "standard-version": "^9.5.0",
     "tsc-watch": "^6.2.0",
     "typescript": "^5.4.5",
-    "typescript-eslint": "^7.12.0"
+    "typescript-eslint": "^7.13.0"
   },
   "config": {
     "commitizen": {

src/api/index.ts

@@ -3,7 +3,6 @@ import Logger from '../utils/Logger.js';
 import express from 'express';
 import dblRoute from './routes/dbl.js';
 import nsfwRouter from './routes/nsfw.js';
-import { API_PORT } from '../utils/Constants.js';
 
 // to start the server
 export const startApi = (data: { voteManager: VoteManager }) => {
@@ -14,5 +13,6 @@ export const startApi = (data: { voteManager: VoteManager }) => {
   app.use(nsfwRouter);
   if (data.voteManager) app.use(dblRoute(data.voteManager));
 
-  app.listen(API_PORT, () => Logger.info(`API listening on port http://localhost:${API_PORT}.`));
+  const port = process.env.PORT;
+  app.listen(port, () => Logger.info(`API listening on port http://localhost:${port}.`));
 };

src/cluster.ts

@@ -14,7 +14,7 @@ import 'dotenv/config';
 
 const clusterManager = new ClusterManager('build/index.js', {
   token: process.env.TOKEN,
-  shardsPerClusters: 2,
+  shardsPerClusters: 5,
   totalClusters: 'auto',
 });
 

src/core/Client.ts

@@ -66,7 +66,7 @@ export default class SuperClient extends Client {
           filter: () => () => true, // Remove all reactions...
         },
       },
-      partials: [Partials.Message],
+      partials: [Partials.Message, Partials.Channel],
       intents: [
         IntentsBitField.Flags.MessageContent,
         IntentsBitField.Flags.Guilds,
@@ -104,19 +104,13 @@ export default class SuperClient extends Client {
     await syncConnectionCache();
     this._connectionCachePopulated = true;
 
-    this.scheduler.addRecurringTask(
-      'populateConnectionCache',
-      60_000 * 5,
-      syncConnectionCache,
-    );
-
-    // store network message timestamps to connectedList every minute
+    this.scheduler.addRecurringTask('populateConnectionCache', 60_000 * 5, syncConnectionCache);
     this.scheduler.addRecurringTask('storeMsgTimestamps', 60 * 1_000, () => {
+      // store network message timestamps to connectedList every minute
       storeMsgTimestamps(messageTimestamps);
       messageTimestamps.clear();
     });
 
-
     await this.login(process.env.TOKEN);
   }
 
@@ -135,7 +129,7 @@ export default class SuperClient extends Client {
   async fetchGuild(guildId: Snowflake): Promise<RemoveMethods<Guild> | undefined> {
     const fetch = (await this.cluster.broadcastEval(
       (client, guildID) => client.guilds.cache.get(guildID),
-      { context: guildId },
+      { guildId, context: guildId },
     )) as Guild[];
 
     return fetch ? SuperClient.resolveEval(fetch) : undefined;

src/scripts/network/helpers.ts

@@ -5,13 +5,17 @@ import {
   ButtonStyle,
   ButtonBuilder,
   ActionRowBuilder,
+  APIMessage,
 } from 'discord.js';
 import db from '../../utils/Db.js';
 import { LINKS, REGEX, emojis } from '../../utils/Constants.js';
 import { censor } from '../../utils/Profanity.js';
 import { broadcastedMessages } from '@prisma/client';
 import { t } from '../../utils/Locale.js';
 
+export type NetworkAPIError = { error: string };
+
+
 /**
  * Retrieves the content of a referred message, which can be either the message's text content or the description of its first embed.
  * If the referred message has no content, returns a default message indicating that the original message contains an attachment.
@@ -172,3 +176,8 @@ export const sendWelcomeMsg = async (message: Message, totalServers: string, hub
     })
     .catch(() => null);
 };
+
+
+export function isNetworkApiError(res: NetworkAPIError | APIMessage | undefined): res is NetworkAPIError {
+  return (res && Object.hasOwn(res, 'error')) === true;
+}

src/scripts/network/runChecks.ts

@@ -117,11 +117,10 @@ export const runChecks = async (
   const { settings, userData, attachmentURL } = opts;
   const isUserBlacklisted = userData.blacklistedFrom.some((b) => b.hubId === hubId);
 
-  if (await isCaughtSpam(message, settings, hubId)) return false;
-  if (containsLinks(message, settings)) message.content = replaceLinks(message.content);
-
   // banned / blacklisted
   if (userData.banMeta?.reason || isUserBlacklisted) return false;
+  if (containsLinks(message, settings)) message.content = replaceLinks(message.content);
+  if (await isCaughtSpam(message, settings, hubId)) return false;
 
   // send a log to the log channel set by the hub
   if (hasProfanity || hasSlurs) {

src/scripts/network/sendBroadcast.ts

@@ -119,7 +119,7 @@ export default (
     catch (e) {
       // return the error and webhook URL to store the message in the db
       return {
-        messageOrError: e.message,
+        messageOrError: { error: e.message },
         webhookURL: connection.webhookURL,
       } as NetworkWebhookSendResult;
     }

src/scripts/network/sendMessage.ts

@@ -1,17 +1,77 @@
-import { APIMessage, WebhookMessageCreateOptions } from 'discord.js';
+import {
+  APIEmbed,
+  APIMessage,
+  WebhookClient,
+  WebhookMessageCreateOptions,
+  isJSONEncodable,
+} from 'discord.js';
+import { NetworkAPIError, isNetworkApiError } from './helpers.js';
+import { encryptMessage, wait } from '../../utils/Utils.js';
 
-export default async (webhookUrl: string, message: WebhookMessageCreateOptions) => {
-  const res = await fetch('https://interchat-networkwebhook.vercel.app/api/send', {
-    method: 'PUT',
-    body: JSON.stringify(message),
+export default async (webhookUrl: string, data: WebhookMessageCreateOptions) => {
+  const webhook = new WebhookClient({ url: webhookUrl });
+  return await webhook.send(data);
+};
+
+const { INTERCHAT_API_URL1, INTERCHAT_API_URL2 } = process.env;
+const urls = [INTERCHAT_API_URL1, INTERCHAT_API_URL2];
+let primaryUrl = urls[0];
+
+const switchUrl = (currentUrl: string) => {
+  if (currentUrl === urls[urls.length - 1]) return urls[0] ?? currentUrl;
+  else return urls[urls.indexOf(currentUrl) + 1] ?? currentUrl;
+};
+export const specialSendMessage = async (
+  webhookUrl: string,
+  data: WebhookMessageCreateOptions,
+  tries = 0,
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  encrypt = true,
+): Promise<NetworkAPIError | APIMessage | undefined> => {
+  const networkKey = process.env.NETWORK_API_KEY;
+  if (!networkKey || !primaryUrl) {
+    throw new Error('NETWORK_API_KEY or INTERCHAT_API_URL(s) env variables missing.');
+  }
+
+  // TODO: Encryption stuff, doesn't work in cf workers :(
+  let embed: APIEmbed = {};
+  if (encrypt) {
+    if (!process.env.NETWORK_ENCRYPT_KEY) throw new Error('Missing encryption key for network.');
+
+    const firstEmbed = data.embeds?.at(0);
+
+    const encryptKey = Buffer.from(process.env.NETWORK_ENCRYPT_KEY, 'base64');
+    const content = data.content;
+    if (encrypt) {
+      if (content) {
+        data.content = encryptMessage(content, encryptKey);
+      }
+      else if (firstEmbed) {
+        embed = isJSONEncodable(firstEmbed) ? firstEmbed.toJSON() : firstEmbed;
+        if (embed.description) {
+          embed.description = encryptMessage(embed.description, encryptKey);
+        }
+      }
+    }
+  }
+
+  // console.log(data);
+  const res = await fetch(primaryUrl, {
+    method: 'POST',
+    body: JSON.stringify({ webhookUrl, data: { ...data, ...embed } }),
     headers: {
-      authorization: `${process.env.NETWORK_API_KEY}`,
-      'x-webhook-url': webhookUrl,
+      authorization: networkKey,
       'Content-Type': 'application/json',
     },
   });
 
-  const resBody = await res.json();
+  const resJson = (await res.json()) as NetworkAPIError | APIMessage | undefined;
+
+  if (isNetworkApiError(resJson) && tries <= 5) {
+    await wait(3000);
+    primaryUrl = switchUrl(primaryUrl);
+    return await specialSendMessage(webhookUrl, data, tries + 1, false);
+  }
 
-  return res.status === 200 ? (resBody.result as APIMessage) : String(resBody.error);
-};
+  return resJson;
+};

src/scripts/network/storeMessageData.ts

@@ -2,10 +2,11 @@ import db from '../../utils/Db.js';
 import { originalMessages } from '@prisma/client';
 import { APIMessage, Message } from 'discord.js';
 import { messageTimestamps, modifyConnections } from '../../utils/ConnectedList.js';
-import { handleError, parseTimestampFromId } from '../../utils/Utils.js';
+import { NetworkAPIError, isNetworkApiError } from './helpers.js';
+import Logger from '../../utils/Logger.js';
 
 export interface NetworkWebhookSendResult {
-  messageOrError: APIMessage | string;
+  messageOrError: APIMessage | NetworkAPIError;
   webhookURL: string;
 }
 
@@ -20,20 +21,21 @@ export default async (
   hubId: string,
   dbReference?: originalMessages | null,
 ) => {
-  const messageDataObj: { channelId: string; messageId: string, createdAt: Date }[] = [];
+  const messageDataObj: { channelId: string; messageId: string; createdAt: Date }[] = [];
   const invalidWebhookURLs: string[] = [];
   const validErrors = ['Invalid Webhook Token', 'Unknown Webhook', 'Missing Permissions'];
 
   // loop through all results and extract message data and invalid webhook urls
   channelAndMessageIds.forEach(({ messageOrError, webhookURL }) => {
-    if (messageOrError && typeof messageOrError !== 'string') {
+    if (!isNetworkApiError(messageOrError)) {
       messageDataObj.push({
         channelId: messageOrError.channel_id,
         messageId: messageOrError.id,
-        createdAt: new Date(parseTimestampFromId(messageOrError.id)),
+        createdAt: new Date(messageOrError.timestamp),
       });
     }
-    else if (validErrors.some((e) => (messageOrError as string).includes(e))) {
+    else if (validErrors.some((e) => messageOrError.error?.includes(e))) {
+      Logger.info('%O', messageOrError); // TODO Remove dis
       invalidWebhookURLs.push(webhookURL);
     }
   });
@@ -53,7 +55,7 @@ export default async (
         hub: { connect: { id: hubId } },
         reactions: {},
       },
-    }).catch(handleError);
+    });
   }
 
   // store message timestamps to push to db later

src/scripts/tasks/pauseIdleConnections.ts

@@ -17,7 +17,7 @@ export default async (manager: ClusterManager) => {
     },
   });
 
-  if (!connections) return;
+  if (connections?.length === 0) return;
 
   const reconnectButtonArr: {
     channelId: Snowflake;
@@ -39,13 +39,13 @@ export default async (manager: ClusterManager) => {
     });
 
     // disconnect the channel
-    await modifyConnection({ channelId }, { lastActive: null, connected: false });
+    await modifyConnection({ channelId }, { connected: false });
   });
 
   const embed = simpleEmbed(
     stripIndents`
     ### ${emojis.timeout} Paused Due to Inactivity
-    Connection to this hub has been stopped. **Click the button** below to resume chatting (or alternatively, \`/connection\`).
+    Connection to this hub has been stopped because no messages were sent for past day. **Click the button** below to resume chatting (or alternatively, \`/connection\`).
     `,
   ).toJSON();
 

src/utils/ConnectedList.ts

@@ -2,7 +2,6 @@ import db from './Db.js';
 import Logger from './Logger.js';
 import { Prisma, connectedList } from '@prisma/client';
 import { Collection } from 'discord.js';
-import { handleError } from './Utils.js';
 import { captureException } from '@sentry/node';
 
 /** 📡 Contains all the **connected** channels from all hubs. */
@@ -72,6 +71,6 @@ export const modifyConnections = async (
 export const storeMsgTimestamps = (data: Collection<string, Date>): void => {
   data.forEach(
     async (lastActive, channelId) =>
-      await modifyConnection({ channelId }, { lastActive }).catch(handleError),
+      await modifyConnection({ channelId }, { lastActive }),
   );
 };

src/utils/Constants.ts

@@ -17,7 +17,6 @@ export const isDevBuild = process.env.NODE_ENV === 'development';
 
 export const PROJECT_VERSION = require('../../package.json').version ?? 'Unknown';
 export const CLIENT_ID = isDevBuild ? '798748015435055134' : '769921109209907241';
-export const API_PORT = isDevBuild ? 3000 : 443;
 export const SUPPORT_SERVER_ID = '770256165300338709';
 export const VOTER_ROLE_ID = '985153241727770655';
 

src/utils/HubLogger/Report.ts

@@ -50,17 +50,16 @@ const genJumpLink = async (
   });
   if (!messageInDb) return null;
 
-
   // fetch the reports server ID from the log channel's ID
   const reportsServerId = SuperClient.resolveEval(
     await client.cluster.broadcastEval(
-      async (cl, ctx) => {
+      async (cl, channelId) => {
         const channel = (await cl.channels
-          .fetch(ctx.reportsChannelId)
+          .fetch(channelId)
           .catch(() => null)) as GuildTextBasedChannel | null;
         return channel?.guild.id;
       },
-      { context: { reportsChannelId } },
+      { context: reportsChannelId },
     ),
   );
 

src/utils/Logger.ts

@@ -1,14 +1,15 @@
 import { createLogger, format, transports } from 'winston';
 import 'source-map-support/register.js';
+
 const custom = format.printf(
   (info) =>
-    `${info.level}: ${info.message} | ${info.timestamp} ${info.stack ? `\n${info.stack}` : ''}`,
+    `${info.timestamp} ${info.level}: ${info.message} ${info.stack ? `\n${info.stack}` : ''}`,
 );
 
 const combinedFormat = format.combine(
   format.errors({ stack: true }),
   format.splat(),
-  format.timestamp({ format: '[on] DD MMMM, YYYY [at] hh:mm:ss.SSS' }),
+  format.timestamp({ format: 'DD/MM/YY-HH:mm:ss' }),
   format((info) => {
     info.level = info.level.toUpperCase();
     return info;

src/utils/NSFWDetection.ts

@@ -1,4 +1,4 @@
-import { API_PORT } from './Constants.js';
+import 'dotenv/config';
 
 export declare type predictionType = {
   className: 'Drawing' | 'Hentai' | 'Neutral' | 'Porn' | 'Sexy';
@@ -10,7 +10,7 @@ export declare type predictionType = {
  * @returns The predictions object
  */
 export const analyzeImageForNSFW = async (imageUrl: string): Promise<predictionType[] | null> => {
-  const res = await fetch(`http://localhost:${API_PORT}/nsfw`, {
+  const res = await fetch(`http://localhost:${process.env.PORT}/nsfw`, {
     method: 'POST',
     headers: { 'Content-Type': 'application/json' },
     body: JSON.stringify({ imageUrl }),