Blacklist manager refactor & New in-memory cache

.env.example

@@ -9,3 +9,4 @@ NETWORK_API_KEY="your network api key" # for posting to global chat (ask devoid)
 NODE_ENV=development # change to production when deploying
 DEBUG=false # set to true to enable debug logging
 PORT=3000 # or anything else for production
+NSFW_AI_MODEL=MobileNetV2 # InceptionV3 for prod

.vscode/settings.json

@@ -7,5 +7,6 @@
     "javascript",
     "typescript"
   ],
-  "eslint.experimental.useFlatConfig": true
+  "eslint.experimental.useFlatConfig": true,
+  "typescript.tsdk": "node_modules/typescript/lib"
 }

package.json

@@ -31,6 +31,7 @@
     "express": "^4.19.2",
     "google-translate-api-x": "^10.6.8",
     "husky": "^9.0.11",
+    "ioredis": "^5.4.1",
     "js-yaml": "^4.1.0",
     "lodash": "^4.17.21",
     "lz-string": "^1.5.0",
@@ -40,12 +41,12 @@
     "winston": "^3.13.0"
   },
   "devDependencies": {
-    "@stylistic/eslint-plugin": "^2.2.1",
+    "@stylistic/eslint-plugin": "^2.3.0",
     "@types/common-tags": "^1.8.4",
     "@types/express": "^4.17.21",
     "@types/js-yaml": "^4.0.9",
-    "@types/lodash": "^4.17.5",
-    "@types/node": "^20.14.2",
+    "@types/lodash": "^4.17.6",
+    "@types/node": "^20.14.9",
     "@types/source-map-support": "^0.5.10",
     "cz-conventional-changelog": "^3.3.0",
     "eslint": "8.57.0",
@@ -54,8 +55,8 @@
     "prisma": "^5.15.0",
     "standard-version": "^9.5.0",
     "tsc-watch": "^6.2.0",
-    "typescript": "^5.4.5",
-    "typescript-eslint": "^7.13.0"
+    "typescript": "^5.5.3",
+    "typescript-eslint": "^7.15.0"
   },
   "config": {
     "commitizen": {

prisma/schema.prisma

@@ -1,5 +1,5 @@
 generator client {
-  provider = "prisma-client-js"
+  provider        = "prisma-client-js"
   previewFeatures = ["tracing"]
 }
 
@@ -8,11 +8,6 @@ datasource db {
   url      = env("DATABASE_URL")
 }
 
-type MessageDataChannelAndMessageIds {
-  channelId String
-  messageId String
-}
-
 type MessageDataReference {
   channelId String
   guildId   String?
@@ -73,10 +68,9 @@ type hubBlacklist {
 }
 
 model blacklistedServers {
-  id         String         @id @default(auto()) @map("_id") @db.ObjectId
-  serverId   String         @unique
-  serverName String
-  hubs       hubBlacklist[]
+  id              String         @id @map("_id") @db.String
+  serverName      String
+  blacklistedFrom hubBlacklist[]
 }
 
 model connectedList {
@@ -146,21 +140,8 @@ model broadcastedMessages {
   originalMsgId String           @db.String
 }
 
-model userBadges {
-  userId String   @id @map("_id")
-  badges String[]
-}
-
-model blacklistedUsers {
-  id       String         @id @default(auto()) @map("_id") @db.ObjectId
-  userId   String         @unique
-  username String
-  hubs     hubBlacklist[]
-}
-
 model userData {
-  id                   String         @id @default(auto()) @map("_id") @db.ObjectId
-  userId               String         @unique
+  id                   String         @id @map("_id") @db.String
   voteCount            Int            @default(0)
   // username is only guarenteed to be set and/or used for blacklisted users
   username             String?

src/api/index.ts

@@ -14,5 +14,5 @@ export const startApi = (data: { voteManager: VoteManager }) => {
   if (data.voteManager) app.use(dblRoute(data.voteManager));
 
   const port = process.env.PORT;
-  app.listen(port, () => Logger.info(`API listening on port http://localhost:${port}.`));
+  app.listen(port, () => Logger.info(`API listening on http://localhost:${port}.`));
 };

src/api/routes/nsfw.ts

@@ -2,15 +2,15 @@ import Logger from '../../utils/Logger.js';
 import { Router } from 'express';
 import { captureException } from '@sentry/node';
 import { node } from '@tensorflow/tfjs-node';
-import { REGEX, isDevBuild } from '../../utils/Constants.js';
+import { REGEX } from '../../utils/Constants.js';
 import { createRequire } from 'module';
 import { NSFWJS } from 'nsfwjs';
 
 const require = createRequire(import.meta.url);
 const { load } = require('nsfwjs');
 
 // InceptionV3 is more accurate but slower and takes up a shit ton of memory
-const nsfwModel: NSFWJS = await load(isDevBuild ? 'MobileNetV2' : 'InceptionV3');
+const nsfwModel: NSFWJS = await load(process.env.NSFW_AI_MODEL);
 const router = Router();
 
 router.post('/nsfw', async (req, res) => {

src/cluster.ts

@@ -1,15 +1,17 @@
 import db from './utils/Db.js';
 import Logger from './utils/Logger.js';
 import Scheduler from './services/SchedulerService.js';
-import syncBotlistStats from './scripts/tasks/syncBotlistStats.js';
-import updateBlacklists from './scripts/tasks/updateBlacklists.js';
-import deleteExpiredInvites from './scripts/tasks/deleteExpiredInvites.js';
-import pauseIdleConnections from './scripts/tasks/pauseIdleConnections.js';
+import syncBotlistStats from './tasks/syncBotlistStats.js';
+import updateBlacklists from './tasks/updateBlacklists.js';
+import storeMsgTimestamps from './tasks/storeMsgTimestamps.js';
+import deleteExpiredInvites from './tasks/deleteExpiredInvites.js';
+import pauseIdleConnections from './tasks/pauseIdleConnections.js';
 import { startApi } from './api/index.js';
 import { isDevBuild } from './utils/Constants.js';
+import { getUsername } from './utils/Utils.js';
 import { VoteManager } from './managers/VoteManager.js';
 import { ClusterManager } from 'discord-hybrid-sharding';
-import { getUsername, wait } from './utils/Utils.js';
+import { getAllConnections } from './utils/ConnectedList.js';
 import 'dotenv/config';
 
 const clusterManager = new ClusterManager('build/index.js', {
@@ -18,52 +20,55 @@ const clusterManager = new ClusterManager('build/index.js', {
   totalClusters: 'auto',
 });
 
-clusterManager.on('clusterCreate', async (cluster) => {
-  // if it is the last cluster
-  if (cluster.id === clusterManager.totalClusters - 1) {
+const voteManager = new VoteManager(clusterManager);
+voteManager.on('vote', async (vote) => {
+  const username = (await getUsername(clusterManager, vote.user)) ?? undefined;
+  await voteManager.incrementUserVote(vote.user, username);
+  await voteManager.addVoterRole(vote.user);
+  await voteManager.announceVote(vote);
+});
+
+startApi({ voteManager });
+
+// spawn clusters and start the api that handles nsfw filter and votes
+clusterManager
+  .spawn({ timeout: -1 })
+  .then(async () => {
     const scheduler = new Scheduler();
 
-    // remove expired blacklists or set new timers for them
-    const serverQuery = await db.blacklistedServers.findMany({
-      where: { hubs: { some: { expires: { isSet: true } } } },
-    });
-    const userQuery = await db.userData.findMany({
-      where: { blacklistedFrom: { some: { expires: { isSet: true } } } },
-    });
+    const blacklistQuery = { where: { blacklistedFrom: { some: { expires: { isSet: true } } } } };
 
-    updateBlacklists(serverQuery, scheduler).catch(Logger.error);
-    updateBlacklists(userQuery, scheduler).catch(Logger.error);
+    // populate cache
+    await db.blacklistedServers.findMany(blacklistQuery);
+    await db.userData.findMany(blacklistQuery);
 
-    // code must be in production to run these tasks
-    if (isDevBuild) return;
-    // give time for shards to connect for these tasks
-    await wait(10_000);
+    updateBlacklists(clusterManager).catch(Logger.error);
+    deleteExpiredInvites().catch(Logger.error);
 
+    if (isDevBuild) return;
     // perform start up tasks
-    syncBotlistStats(clusterManager).catch(Logger.error);
-    deleteExpiredInvites().catch(Logger.error);
+    const serverCount = (await clusterManager.fetchClientValues('guilds.cache.size')).reduce(
+      (p: number, n: number) => p + n,
+      0,
+    );
+
+    syncBotlistStats({ serverCount, shardCount: clusterManager.totalShards }).catch(Logger.error);
     pauseIdleConnections(clusterManager).catch(Logger.error);
 
+    // store network message timestamps to connectedList every minute
+    scheduler.addRecurringTask('storeMsgTimestamps', 60 * 1_000, () => storeMsgTimestamps);
     scheduler.addRecurringTask('deleteExpiredInvites', 60 * 60 * 1000, deleteExpiredInvites);
-    scheduler.addRecurringTask('pauseIdleConnections', 60 * 60 * 1000, () =>
-      pauseIdleConnections(clusterManager),
+    scheduler.addRecurringTask('populateConnectionCache', 5 * 60 * 1000, () =>
+      getAllConnections({ connected: true }),
+    );
+    scheduler.addRecurringTask('deleteExpiredBlacklists', 10 * 1000, () =>
+      updateBlacklists(clusterManager),
     );
     scheduler.addRecurringTask('syncBotlistStats', 10 * 60 * 10_000, () =>
-      syncBotlistStats(clusterManager),
+      syncBotlistStats({ serverCount, shardCount: clusterManager.totalShards }),
     );
-  }
-});
-
-const voteManager = new VoteManager(clusterManager);
-voteManager.on('vote', async (vote) => {
-  const username = (await getUsername(clusterManager, vote.user)) ?? undefined;
-  await voteManager.incrementUserVote(vote.user, username);
-  await voteManager.addVoterRole(vote.user);
-  await voteManager.announceVote(vote);
-});
-
-// spawn clusters and start the api that handles nsfw filter and votes
-clusterManager
-  .spawn({ timeout: -1 })
-  .then(() => startApi({ voteManager }))
+    scheduler.addRecurringTask('pauseIdleConnections', 60 * 60 * 1000, () =>
+      pauseIdleConnections(clusterManager),
+    );
+  })
   .catch(Logger.error);

src/commands/context-menu/blacklist.ts

@@ -59,6 +59,14 @@ export default class Blacklist extends BaseCommand {
       return;
     }
 
+    if (messageInDb.originalMsg.authorId === interaction.user.id) {
+      await interaction.reply({
+        content: '<a:nuhuh:1256859727158050838> Nuh uh! You\'re stuck with us.',
+        ephemeral: true,
+      });
+      return;
+    }
+
     const server = await interaction.client.fetchGuild(messageInDb.originalMsg.serverId);
     const user = await interaction.client.users.fetch(messageInDb.originalMsg.authorId);
 
@@ -177,7 +185,7 @@ export default class Blacklist extends BaseCommand {
 
     const reason = interaction.fields.getTextInputValue('reason');
     const duration = parse(interaction.fields.getTextInputValue('duration'));
-    const expires = duration ? new Date(Date.now() + duration) : undefined;
+    const expires = duration ? new Date(Date.now() + duration) : null;
 
     const successEmbed = new EmbedBuilder().setColor('Green').addFields(
       {
@@ -192,36 +200,40 @@ export default class Blacklist extends BaseCommand {
       },
     );
 
-    const blacklistManager = interaction.client.blacklistManager;
+    const { userManager } = interaction.client;
 
     // user blacklist
     if (customId.suffix === 'user') {
       const user = await interaction.client.users.fetch(originalMsg.authorId).catch(() => null);
+
+      if (!user) {
+        await interaction.reply({
+          embeds: [
+            simpleEmbed(
+              `${emojis.neutral} Unable to fetch user. They may have deleted their account?`,
+            ),
+          ],
+          ephemeral: true,
+        });
+        return;
+      }
+
       successEmbed.setDescription(
         t(
           { phrase: 'blacklist.user.success', locale },
           { username: user?.username ?? 'Unknown User', emoji: emojis.tick },
         ),
       );
 
-      await blacklistManager.addUserBlacklist(
-        originalMsg.hubId,
-        originalMsg.authorId,
+      await userManager.addBlacklist({ id: user.id, name: user.username }, originalMsg.hubId, {
         reason,
-        interaction.user.id,
+        moderatorId: interaction.user.id,
         expires,
-      );
+      });
 
-      if (expires) {
-        blacklistManager.scheduleRemoval('user', originalMsg.authorId, originalMsg.hubId, expires);
-      }
       if (user) {
-        blacklistManager
-          .notifyBlacklist('user', originalMsg.authorId, {
-            hubId: originalMsg.hubId,
-            expires,
-            reason,
-          })
+        userManager
+          .sendNotification({ target: user, hubId: originalMsg.hubId, expires, reason })
           .catch(() => null);
 
         await logBlacklist(originalMsg.hubId, interaction.client, {
@@ -241,6 +253,7 @@ export default class Blacklist extends BaseCommand {
 
     // server blacklist
     else {
+      const { serverBlacklists } = interaction.client;
       const server = await interaction.client.fetchGuild(originalMsg.serverId);
 
       successEmbed.setDescription(
@@ -250,34 +263,25 @@ export default class Blacklist extends BaseCommand {
         ),
       );
 
-      await blacklistManager.addServerBlacklist(
-        originalMsg.serverId,
+      await serverBlacklists.addBlacklist(
+        { name: server?.name ?? 'Unknown Server', id: originalMsg.serverId },
         originalMsg.hubId,
-        reason,
-        interaction.user.id,
-        expires,
+        {
+          reason,
+          moderatorId: interaction.user.id,
+          expires,
+        },
       );
 
       // Notify server of blacklist
-      await blacklistManager.notifyBlacklist('server', originalMsg.serverId, {
+      await serverBlacklists.sendNotification({
+        target: { id: originalMsg.serverId },
         hubId: originalMsg.hubId,
         expires,
         reason,
       });
 
-      if (expires) {
-        blacklistManager.scheduleRemoval(
-          'server',
-          originalMsg.serverId,
-          originalMsg.hubId,
-          expires,
-        );
-      }
-
-      await deleteConnections({
-        serverId: originalMsg.serverId,
-        hubId: originalMsg.hubId,
-      });
+      await deleteConnections({ serverId: originalMsg.serverId, hubId: originalMsg.hubId });
 
       if (server) {
         await logBlacklist(originalMsg.hubId, interaction.client, {

src/commands/context-menu/deleteMsg.ts

@@ -10,6 +10,7 @@ import { REGEX, emojis } from '../../utils/Constants.js';
 import { t } from '../../utils/Locale.js';
 import { logMsgDelete } from '../../utils/HubLogger/ModLogs.js';
 import { captureException } from '@sentry/node';
+import { getAllConnections } from '../../utils/ConnectedList.js';
 
 export default class DeleteMessage extends BaseCommand {
   readonly data: RESTPostAPIApplicationCommandsJSONBody = {
@@ -80,10 +81,10 @@ export default class DeleteMessage extends BaseCommand {
 
     let passed = 0;
 
+    const allConnections = await getAllConnections();
+
     for await (const dbMsg of originalMsg.broadcastMsgs) {
-      const connection = interaction.client.connectionCache.find(
-        (c) => c.channelId === dbMsg.channelId,
-      );
+      const connection = allConnections?.find((c) => c.channelId === dbMsg.channelId);
 
       if (!connection) break;