improve local store serialization, closes #406

* Turn "msgpack" decoding to asynchronous mode.
* Serialize/deserialize mails by portions, one by one, so "msgpack" never faces too huge bunch of bytes. This also means that memory use peaks get reduced (one by one portions processing).
* Compress the stored data (gzip).
* Compress the mails "raw" and "body" props (lzutf8).
* Double the default app starting memory related arguments.

electron-builder.yml

@@ -54,7 +54,7 @@ linux:
   desktop:
     StartupWMClass: electron-mail
   executableArgs:
-    - '--js-flags="--max-old-space-size=6144"'
+    - '--js-flags="--max-old-space-size=12288"'
 
 snap:
   confinement: strict

package.json

@@ -123,11 +123,14 @@
     "electron-rpc-api": "8.0.0",
     "electron-unhandled": "3.0.2",
     "fast-glob": "3.2.5",
+    "fs-backwards-stream": "1.0.0",
     "fs-extra": "10.0.0",
     "fs-json-store": "7.0.2",
     "fs-json-store-encryption-adapter": "3.0.3",
+    "fs-write-stream-atomic": "1.0.10",
     "js-base64": "3.6.1",
     "keytar": "7.7.0",
+    "lzutf8": "0.6.0",
     "node-fetch": "2.6.1",
     "oboe": "2.1.5",
     "os-locale": "5.0.0",
@@ -181,6 +184,7 @@
     "@types/randomstring": "1.1.6",
     "@types/react": "17.0.8",
     "@types/react-router": "5.1.14",
+    "@types/readable-stream": "2.3.10",
     "@types/rimraf": "3.0.0",
     "@types/sanitize-html": "2.3.1",
     "@types/semver": "7.3.6",

patches/patch-package/app-builder-lib+22.11.5.patch

@@ -1,13 +1,13 @@
 diff --git a/node_modules/app-builder-lib/templates/nsis/include/installer.nsh b/node_modules/app-builder-lib/templates/nsis/include/installer.nsh
-index 31a1416..572f371 100644
+index 34e91df..27f12cd 100644
 --- a/node_modules/app-builder-lib/templates/nsis/include/installer.nsh
 +++ b/node_modules/app-builder-lib/templates/nsis/include/installer.nsh
 @@ -174,7 +174,7 @@
        !insertmacro cleanupOldMenuDirectory
        !insertmacro createMenuDirectory
 
 -      CreateShortCut "$newStartMenuLink" "$appExe" "" "$appExe" 0 "" "" "${APP_DESCRIPTION}"
-+      CreateShortCut "$newStartMenuLink" "$appExe" "--js-flags=$\"--max-old-space-size=6144$\"" "$appExe" 0 "" "" "${APP_DESCRIPTION}"
++      CreateShortCut "$newStartMenuLink" "$appExe" "--js-flags=$\"--max-old-space-size=12288$\"" "$appExe" 0 "" "" "${APP_DESCRIPTION}"
        # clear error (if shortcut already exists)
        ClearErrors
        WinShell::SetLnkAUMI "$newStartMenuLink" "${APP_ID}"
@@ -16,7 +16,7 @@ index 31a1416..572f371 100644
        # Shortcuts will be recreated.
        ${if} $keepShortcuts == "false"
 -        CreateShortCut "$newDesktopLink" "$appExe" "" "$appExe" 0 "" "" "${APP_DESCRIPTION}"
-+        CreateShortCut "$newDesktopLink" "$appExe" "--js-flags=$\"--max-old-space-size=6144$\"" "$appExe" 0 "" "" "${APP_DESCRIPTION}"
++        CreateShortCut "$newDesktopLink" "$appExe" "--js-flags=$\"--max-old-space-size=12288$\"" "$appExe" 0 "" "" "${APP_DESCRIPTION}"
          ClearErrors
          WinShell::SetLnkAUMI "$newDesktopLink" "${APP_ID}"
        # The keepShortcuts mechanism IS enabled.
@@ -25,7 +25,7 @@ index 31a1416..572f371 100644
        ${orIfNot} ${FileExists} "$oldDesktopLink"
          ${ifNot} ${isUpdated}
 -          CreateShortCut "$newDesktopLink" "$appExe" "" "$appExe" 0 "" "" "${APP_DESCRIPTION}"
-+          CreateShortCut "$newDesktopLink" "$appExe" "--js-flags=$\"--max-old-space-size=6144$\"" "$appExe" 0 "" "" "${APP_DESCRIPTION}"
++          CreateShortCut "$newDesktopLink" "$appExe" "--js-flags=$\"--max-old-space-size=12288$\"" "$appExe" 0 "" "" "${APP_DESCRIPTION}"
            ClearErrors
            WinShell::SetLnkAUMI "$newDesktopLink" "${APP_ID}"
          ${endIf}

src/@types/fs-backwards-stream/index.d.ts

@@ -0,0 +1,8 @@
+declare module "fs-backwards-stream" {
+    import {Readable} from "stream";
+
+    export default function(
+        file: string,
+        opts?: { block?: number },
+    ): Pick<Readable, "on" | "once" | "destroy">;
+}

src/@types/fs-write-stream-atomic/index.d.ts

@@ -0,0 +1,17 @@
+declare module "fs-write-stream-atomic" {
+    import {Writable, WritableOptions} from "stream";
+    import {WritableStateOptions} from "readable-stream";
+
+    class WriteStreamAtomic extends Writable {
+        public constructor(
+            path: string,
+            options?: NoExtraProps<WritableOptions & WritableStateOptions & {
+                chown?: { uid: number, guid: number }
+                encoding?: BufferEncoding
+                flags?: import("fs").OpenMode
+                mode?: import("fs").Mode
+            }>);
+    }
+
+    export = WriteStreamAtomic;
+}

src/electron-main/api/endpoints-builders/database/export/service.ts

@@ -1,14 +1,14 @@
 import UUID from "pure-uuid";
+import fsAsync from "fs/promises";
 import path from "path";
 import truncateStringLength from "truncate-utf8-bytes";
 import {Base64} from "js-base64";
 import {deserializeError} from "serialize-error";
 
-import fsAsync from "fs/promises";
 import {DbExportMailAttachmentItem} from "src/electron-main/api/endpoints-builders/database/export/const";
 import {File, Mail, MailAddress} from "src/shared/model/database";
 import {PACKAGE_NAME} from "src/shared/constants";
-import {parseProtonRestModel} from "src/shared/util";
+import {parseProtonRestModel, readMailBody} from "src/shared/entity-util";
 
 const eol = "\r\n";
 
@@ -161,7 +161,7 @@ const contentBuilders: Readonly<Record<"eml" | "json", (
         const body = Base64.encode(
             mail.failedDownload
                 ? mail.failedDownload.errorMessage
-                : mail.body,
+                : readMailBody(mail)
         );
         const rawMail = parseProtonRestModel(mail);
         const lines = [
@@ -221,7 +221,7 @@ const contentBuilders: Readonly<Record<"eml" | "json", (
         return JSON.stringify(
             {
                 ...rawMail,
-                Body: mail.body,
+                Body: readMailBody(mail),
                 EncryptedBody: rawMail.Body,
                 Attachments: attachmentsContent
                     ? rawMail.Attachments.map((attachment, index) => {

src/electron-main/api/endpoints-builders/database/index.ts

@@ -17,6 +17,7 @@ import {curryFunctionMembers, isEntityUpdatesPatchNotEmpty} from "src/shared/uti
 import {narrowIndexActionPayload} from "./indexing/service";
 import {patchMetadata} from "src/electron-main/database/util";
 import {prepareFoldersView} from "./folders-view";
+import {readMailBody} from "src/shared/entity-util";
 import {validateEntity} from "src/electron-main/database/validation";
 
 const _logger = curryFunctionMembers(electronLog, __filename);
@@ -240,7 +241,9 @@ export async function buildEndpoints(ctx: Context): Promise<Pick<IpcMainApiEndpo
             return {
                 ...omit(mail, ["body"]),
                 // TODO test "dbGetAccountMail" setting "mail.body" through the "sanitizeHtml" call
-                body: sanitizeHtml(mail.body),
+                body: sanitizeHtml(
+                    readMailBody(mail),
+                ),
             };
         },
     };

src/electron-main/api/endpoints-builders/database/indexing/service.ts

@@ -11,6 +11,7 @@ import {IPC_MAIN_API_DB_INDEXER_REQUEST$, IPC_MAIN_API_DB_INDEXER_RESPONSE$,} fr
 import {IPC_MAIN_API_DB_INDEXER_REQUEST_ACTIONS, IPC_MAIN_API_DB_INDEXER_RESPONSE_ACTIONS,} from "src/shared/api/main";
 import {curryFunctionMembers} from "src/shared/util";
 import {hrtimeDuration} from "src/electron-main/util";
+import {readMailBody} from "src/shared/entity-util";
 
 const logger = curryFunctionMembers(electronLog, __filename);
 
@@ -29,7 +30,12 @@ export const narrowIndexActionPayload: (
         return {
             key,
             remove,
-            add: add.map((mail) => pick(mail, fieldsToIndex)),
+            add: add.map((mail) => {
+                return {
+                    ...pick(mail, fieldsToIndex),
+                    [((prop: keyof Pick<typeof mail, "body">) => prop)("body")]: readMailBody(mail),
+                };
+            }),
         };
     };
 

src/electron-main/api/endpoints-builders/database/search/service.ts

@@ -11,7 +11,8 @@ import {
     fillFoldersAndReturnRootConversationNodes,
     splitAndFormatAndFillSummaryFolders,
 } from "src/electron-main/api/endpoints-builders/database/folders-view";
-import {parseProtonRestModel, walkConversationNodesTree} from "src/shared/util";
+import {parseProtonRestModel, readMailBody} from "src/shared/entity-util";
+import {walkConversationNodesTree} from "src/shared/util";
 
 export function searchRootConversationNodes(
     account: DeepReadonly<FsDbAccount>,
@@ -102,7 +103,7 @@ export const secondSearchStep = async (
                 const serializedMailCodePart = JSON.stringify(
                     JSON.stringify({
                         ...parsedRawMail,
-                        Body: mail.body,
+                        Body: readMailBody(mail),
                         EncryptedBody: parsedRawMail.Body,
                         ...(mail.failedDownload && {_BodyDecryptionFailed: true}),
                         Folders: formFoldersForQuickJSEvaluation(folders, LABEL_TYPE.MESSAGE_FOLDER),

src/electron-main/api/index.ts

@@ -129,8 +129,8 @@ export const initApi = async (ctx: Context): Promise<IpcMainApiEndpoints> => {
         },
 
         // eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types
-        async logout() {
-            if (ctx.keytarSupport) {
+        async logout({skipKeytarProcessing}) {
+            if (!skipKeytarProcessing && ctx.keytarSupport) {
                 await deletePassword();
             }
 

src/electron-main/context.ts

@@ -271,20 +271,8 @@ export function initContext(
                 },
             } as const;
             return {
-                db: new Database(
-                    {
-                        file: path.join(locations.userDataDir, "database.bin"),
-                        encryption,
-                    },
-                    storeFs,
-                ),
-                sessionDb: new Database(
-                    {
-                        file: path.join(locations.userDataDir, "database-session.bin"),
-                        encryption,
-                    },
-                    storeFs,
-                ),
+                db: new Database({file: path.join(locations.userDataDir, "database.bin"), encryption}),
+                sessionDb: new Database({file: path.join(locations.userDataDir, "database-session.bin"), encryption}),
             };
         })(),
         ...((): NoExtraProps<Pick<Context, "sessionStorage">> => {

src/electron-main/database/entity/base.ts

@@ -1,4 +1,4 @@
-import {IsJSON, IsNotEmpty, IsString} from "class-validator";
+import {IsIn, IsNotEmpty, IsOptional, IsString} from "class-validator";
 
 import * as Model from "src/shared/model/database";
 
@@ -7,11 +7,14 @@ export abstract class Entity implements Model.Entity {
     @IsString()
     pk!: Model.Entity["pk"];
 
-    @IsJSON()
     @IsNotEmpty()
     @IsString()
     raw!: Model.Entity["raw"];
 
+    @IsIn(["lzutf8"])
+    @IsOptional()
+    rawCompression!: Model.Mail["rawCompression"];
+
     @IsNotEmpty()
     @IsString()
     id!: Model.Entity["id"];

src/electron-main/database/entity/mail.ts

@@ -69,6 +69,10 @@ export class Mail extends Entity implements Model.Mail {
     @IsString()
     body!: Model.Mail["body"];
 
+    @IsIn(["lzutf8"])
+    @IsOptional()
+    bodyCompression!: Model.Mail["bodyCompression"];
+
     @IsNotEmpty()
     @ValidateNested()
     @Type(() => MailAddress)