feat: optimize SQLite WebSockets with msgpackr and Buffer type check …

…fix in recursively parse fn, switch to safe-stable-stringify instead of fast-safe-stringify for majority of use cases (except logger which requires build and has acorn error otherwise), added 50 MB email test, added round trip time to payload, moved attachment body BLOB to last column as optimization (until we move it to foreign db ref), optimize ping/pong conditional check

app/controllers/web/admin/dashboard.js

@@ -10,7 +10,7 @@ const memoize = require('memoizee');
 const ms = require('ms');
 const numeral = require('numeral');
 const revHash = require('rev-hash');
-const safeStringify = require('fast-safe-stringify');
+const safeStringify = require('safe-stable-stringify');
 const titleize = require('titleize');
 
 const config = require('#config');

app/models/attachments.js

@@ -48,13 +48,29 @@ const Attachments = new mongoose.Schema(
     contentType: { type: String, required: true },
     transferEncoding: { type: String, required: true },
     lineCount: { type: Number, required: true },
-    body: { type: Buffer, required: true },
     counter: { type: Number, required: true }, // metadata.c
     counterUpdated: { type: Date, required: true }, // metadata.cu
-    size: { type: Number, required: true } // metadata.esize (attachment body length)
+    size: { type: Number, required: true }, // metadata.esize (attachment body length)
     // unlike wildduck we don't use these right now:
     // - `metadata.decoded` (Boolean)
     // - `metadata.lineLen` (Number)
+
+    //
+    //       <https://sqlite-users.sqlite.narkive.com/Q4txMI8t/effect-of-blobs-on-performance#post3>
+    //
+    //       Quote from the author of SQLite:
+    //
+    //       > Here's a hint though - make the BLOB columns the last column in
+    //       > your tables. Or even store the BLOBs in a separate table which
+    //       > only has two columns: an integer primary key and the blob itself,
+    //       > and then access the BLOB content using a join if you need to.
+    //       > If you put various small integer fields after the BLOB, then
+    //       > SQLite has to scan through the entire BLOB content (following
+    //       > the linked list of disk pages) to get to the integer fields at
+    //       > the end, and that definitely can slow you down.
+    //       > - D. Richard Hipp
+    //
+    body: { type: Buffer, required: true }
   },
   dummySchemaOptions
 );

app/models/logs.js

@@ -23,7 +23,7 @@ const mongooseCommonPlugin = require('mongoose-common-plugin');
 const pMap = require('p-map');
 const parseErr = require('parse-err');
 const revHash = require('rev-hash');
-const safeStringify = require('fast-safe-stringify');
+const safeStringify = require('safe-stable-stringify');
 const sharedConfig = require('@ladjs/shared-config');
 const splitLines = require('split-lines');
 const twilio = require('twilio');

app/models/threads.js

@@ -19,7 +19,7 @@ const Database = require('better-sqlite3-multiple-ciphers');
 const MessageHandler = require('wildduck/lib/message-handler');
 const _ = require('lodash');
 const mongoose = require('mongoose');
-const safeStringify = require('fast-safe-stringify');
+const safeStringify = require('safe-stable-stringify');
 const validationErrorTransform = require('mongoose-validation-error-transform');
 const { Builder } = require('json-sql');
 

app/models/users.js

@@ -16,7 +16,7 @@ const mongooseCommonPlugin = require('mongoose-common-plugin');
 const mongooseOmitCommonFields = require('mongoose-omit-common-fields');
 const ms = require('ms');
 const passportLocalMongoose = require('passport-local-mongoose');
-const safeStringify = require('fast-safe-stringify');
+const safeStringify = require('safe-stable-stringify');
 const sanitizeHtml = require('sanitize-html');
 const striptags = require('striptags');
 const validator = require('validator');

config/koa-cash.js

@@ -8,7 +8,7 @@ const { Buffer } = require('node:buffer');
 const ms = require('ms');
 const nodeGzip = require('node-gzip');
 const pTimeout = require('p-timeout');
-const safeStringify = require('fast-safe-stringify');
+const safeStringify = require('safe-stable-stringify');
 
 const logger = require('#helpers/logger');
 

helpers/create-mta-sts-cache.js

@@ -3,7 +3,7 @@
  * SPDX-License-Identifier: BUSL-1.1
  */
 
-const safeStringify = require('fast-safe-stringify');
+const safeStringify = require('safe-stable-stringify');
 
 const logger = require('./logger');
 

helpers/create-websocket-as-promised.js

@@ -14,7 +14,6 @@ const mongoose = require('mongoose');
 const ms = require('ms');
 const pRetry = require('p-retry');
 const revHash = require('rev-hash');
-const safeStringify = require('fast-safe-stringify');
 const { WebSocket } = require('ws');
 
 const config = require('#config');
@@ -25,6 +24,7 @@ const parseError = require('#helpers/parse-error');
 const recursivelyParse = require('#helpers/recursively-parse');
 const refineAndLogError = require('#helpers/refine-and-log-error');
 const { encrypt } = require('#helpers/encrypt-decrypt');
+const { encoder, decoder } = require('#helpers/encoder-decoder');
 
 // <https://github.com/partykit/partykit/tree/main/packages/partysocket>
 // <https://github.com/partykit/partykit/issues/536>
@@ -80,15 +80,16 @@ function createWebSocketAsPromised(options = {}) {
           maxPayload: 0, // disable max payload size
           auth,
           rejectUnauthorized: config.env !== 'production'
+          // perMessageDeflate: false,
+          // headers: {
+          //   authorization: 'Basic ' + Buffer.from(auth).toString('base64')
+          // }
         }),
         debug: config.env === 'development'
       });
     },
-    packMessage: (data) => safeStringify(data),
-    unpackMessage(data) {
-      if (typeof data !== 'string') return data;
-      return JSON.parse(data);
-    },
+    packMessage: (data) => encoder.pack(data),
+    unpackMessage: (data) => decoder.unpack(data),
     attachRequestId(data, id) {
       return {
         id,
@@ -147,7 +148,7 @@ function createWebSocketAsPromised(options = {}) {
       // (for initial connection)
       if (!wsp.isOpened) {
         await pRetry(() => wsp.open(), {
-          retries: 9, // in case the default in node-retry changes
+          retries: config.env === 'production' ? 9 : 1, // in case the default in node-retry changes
           onFailedAttempt(err) {
             err.isCodeBug = true;
             // <https://github.com/vitalets/websocket-as-promised/issues/47>
@@ -167,6 +168,7 @@ function createWebSocketAsPromised(options = {}) {
       // (e.g. in case connection disconnected and no response was made)
       const response = await pRetry(
         () => {
+          data.date = Date.now();
           return wsp.sendRequest(data, {
             timeout:
               typeof data.timeout === 'number' &&

helpers/encoder-decoder.js

@@ -0,0 +1,75 @@
+/**
+ * Copyright (c) Forward Email LLC
+ * SPDX-License-Identifier: BUSL-1.1
+ */
+
+const { isNativeAccelerationEnabled, Packr, Unpackr } = require('msgpackr');
+
+const logger = require('./logger');
+
+// <https://github.com/kriszyp/msgpackr?tab=readme-ov-file#options>
+const options = { moreTypes: true };
+
+const packr = new Packr(options);
+const unpackr = new Unpackr(options);
+
+// <https://github.com/kriszyp/msgpackr?tab=readme-ov-file#native-acceleration>
+if (!isNativeAccelerationEnabled)
+  logger.fatal(
+    new TypeError('Native acceleration is not enabled for msgpackr')
+  );
+
+/*
+//
+// NOTE: @msgpack/msgpack is slower than msgpackr
+//
+const { Encoder, Decoder, ExtensionCodec } = require('@msgpack/msgpack');
+
+// https://github.com/msgpack/msgpack-javascript?tab=readme-ov-file#extension-types
+const extensionCodec = new ExtensionCodec();
+
+const encoder = new Encoder({ useBigInt64: true, extensionCodec });
+const decoder = new Decoder({ useBigInt64: true, extensionCodec });
+
+// Set
+const SET_EXT_TYPE = 0; // Any in 0-127
+extensionCodec.register({
+  type: SET_EXT_TYPE,
+  encode(object) {
+    if (object instanceof Set) {
+      console.log('object was a set', object);
+      return encoder.encode([...object]);
+    }
+
+    return null;
+  },
+  decode(data) {
+    const array = decoder.decode(data);
+    return new Set(array);
+  }
+});
+
+// Map
+const MAP_EXT_TYPE = 1; // Any in 0-127
+extensionCodec.register({
+  type: MAP_EXT_TYPE,
+  encode(object) {
+    if (object instanceof Map) {
+      console.log('object was a map', object);
+      return encoder.encode([...object]);
+    }
+
+    return null;
+  },
+  decode(data) {
+    const array = decoder.decode(data);
+    return new Map(array);
+  }
+});
+*/
+
+// NOTE: this is extremely slow
+// const encoder = { encode: JSON.stringify };
+// const decoder = { decode: JSON.parse };
+
+module.exports = { encoder: packr, decoder: unpackr };

helpers/get-fingerprint.js

@@ -5,7 +5,7 @@
 
 const isSANB = require('is-string-and-not-blank');
 const revHash = require('rev-hash');
-const safeStringify = require('fast-safe-stringify');
+const safeStringify = require('safe-stable-stringify');
 
 //
 // generate a fingerprint for the email (returns a short md5 hash)

helpers/imap-notifier.js

@@ -19,7 +19,7 @@ const Axe = require('axe');
 const Database = require('better-sqlite3-multiple-ciphers');
 const _ = require('lodash');
 const ms = require('ms');
-const safeStringify = require('fast-safe-stringify');
+const safeStringify = require('safe-stable-stringify');
 
 const IMAPError = require('#helpers/imap-error');
 const Journals = require('#models/journals');

helpers/imap/on-append.js

@@ -361,9 +361,14 @@ async function onAppend(path, flags, date, raw, session, fn) {
     // prepare text for indexing
     let text = '';
     if (maildata.text) {
+      //
+      // NOTE: without `slice(0, 1MB)` it would output following and cause max callstack exceeded error
+      //
+      //       > Input length 49999999 is above allowed limit of 16777216. Truncating without ellipsis.
+      //
       // replace line breaks for consistency
-      text = splitLines(maildata.text).join(' ').trim();
-      // convert and remove unnecessary things
+      text = splitLines(maildata.text).join(' ').trim().slice(0, 1048576); // 1 MB
+      // convert and remove unnecessary HTML
       text = convert(text, {
         wordwrap: false,
         selectors: [
@@ -484,6 +489,7 @@ async function onAppend(path, flags, date, raw, session, fn) {
       session
     });
 
+    // TODO: we could probably remove this or make it happen after
     // update storage
     try {
       const size = await this.wsp.request({

helpers/imap/on-fetch.js

@@ -16,7 +16,6 @@
 const _ = require('lodash');
 const getStream = require('get-stream');
 const mongoose = require('mongoose');
-const safeStringify = require('fast-safe-stringify');
 const tools = require('wildduck/lib/tools');
 const { Builder } = require('json-sql');
 const { IMAPConnection } = require('wildduck/imap-core/lib/imap-connection');
@@ -112,7 +111,7 @@ async function getMessages(instance, session, server, opts = {}) {
   }
 
   // converts objectids -> strings and arrays/json appropriately
-  const condition = JSON.parse(safeStringify(pageQuery));
+  const condition = JSON.parse(JSON.stringify(pageQuery));
 
   // TODO: `condition` may need further refined for accuracy (e.g. see `prepareQuery`)
 

helpers/imap/on-list.js

@@ -42,8 +42,10 @@ async function onList(query, session, fn) {
     await this.refreshSession(session, 'LIST');
 
     const mailboxes = await Mailboxes.find(this, session, {});
-
-    fn(null, mailboxes);
+    fn(
+      null,
+      mailboxes.map((m) => m.toObject())
+    );
   } catch (err) {
     // NOTE: wildduck uses `imapResponse` so we are keeping it consistent
     if (err.imapResponse) {

helpers/imap/on-lsub.js

@@ -45,7 +45,10 @@ async function onLsub(query, session, fn) {
       subscribed: true
     });
 
-    fn(null, mailboxes);
+    fn(
+      null,
+      mailboxes.map((m) => m.toObject())
+    );
   } catch (err) {
     // NOTE: wildduck uses `imapResponse` so we are keeping it consistent
     if (err.imapResponse) {

helpers/imap/on-search.js

@@ -17,7 +17,6 @@ const { Buffer } = require('node:buffer');
 
 const _ = require('lodash');
 const tools = require('wildduck/lib/tools');
-const safeStringify = require('fast-safe-stringify');
 const { Builder } = require('json-sql');
 
 const IMAPError = require('#helpers/imap-error');
@@ -642,7 +641,7 @@ async function onSearch(mailboxId, options, session, fn) {
     if ($and.length > 0) query.$and = $and;
 
     // converts objectids -> strings and arrays/json appropriately
-    const condition = JSON.parse(safeStringify(query));
+    const condition = JSON.parse(JSON.stringify(query));
 
     const sql = builder.build({
       type: 'select',

helpers/imap/on-store.js

@@ -15,7 +15,6 @@
 
 const imapTools = require('wildduck/imap-core/lib/imap-tools');
 const ms = require('ms');
-const safeStringify = require('fast-safe-stringify');
 const tools = require('wildduck/lib/tools');
 const { Builder } = require('json-sql');
 
@@ -126,7 +125,7 @@ async function onStore(mailboxId, update, session, fn) {
     else query.uid = tools.checkRangeQuery(update.messages);
 
     // converts objectids -> strings and arrays/json appropriately
-    const condition = JSON.parse(safeStringify(query));
+    const condition = JSON.parse(JSON.stringify(query));
 
     // TODO: `condition` may need further refined for accuracy (e.g. see `prepareQuery`)
 

helpers/index.js

@@ -20,6 +20,7 @@ const syncPayPalSubscriptionPaymentsByUser = require('./sync-paypal-subscription
 const syncStripePaymentIntent = require('./sync-stripe-payment-intent');
 const toObject = require('./to-object');
 const { encrypt, decrypt } = require('./encrypt-decrypt');
+const { encoder, decoder } = require('./encoder-decoder');
 const { paypalAgent, paypal } = require('./paypal');
 // TODO: create an npm package for this and then add it in smtp code too
 const combineErrors = require('./combine-errors');
@@ -90,6 +91,8 @@ module.exports = {
   decrypt,
   email,
   encrypt,
+  encoder,
+  decoder,
   getEmailLocals,
   i18n,
   isErrorConstructorName,

helpers/mongoose-to-sqlite.js

@@ -9,7 +9,7 @@ const Database = require('better-sqlite3-multiple-ciphers');
 const _ = require('lodash');
 const isSANB = require('is-string-and-not-blank');
 const mongoose = require('mongoose');
-const safeStringify = require('fast-safe-stringify');
+const safeStringify = require('safe-stable-stringify');
 const { Builder } = require('json-sql');
 
 const env = require('#config/env');

helpers/on-auth.js

@@ -10,7 +10,7 @@ const isSANB = require('is-string-and-not-blank');
 const ms = require('ms');
 const pify = require('pify');
 const revHash = require('rev-hash');
-const safeStringify = require('fast-safe-stringify');
+const safeStringify = require('safe-stable-stringify');
 const { IMAPServer } = require('wildduck/imap-core');
 const { isEmail } = require('validator');