diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5a8183f24..d200cb3c0 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -13,7 +13,7 @@ jobs: steps: # TODO use YAML anchors feature when it gets supported - { uses: actions/checkout@v2 } - - { uses: pnpm/action-setup@v2, with: { version: 7.5.2 } } + - { uses: pnpm/action-setup@v2, with: { version: "latest" } } - { name: pnpm patch, if: runner.os == 'Windows', run: ./scripts/ci/github/patch-pnpm.ps1 } - { uses: actions/setup-node@v2, with: { node-version: "${{ env.ELECTRON_MAIL_NODE_VERSION }}", cache: "pnpm" } } - { name: system setup, if: runner.os == 'Linux', run: ./scripts/ci/github/system-setup-linux.sh } @@ -40,7 +40,7 @@ jobs: steps: # TODO use YAML anchors feature when it gets supported - { uses: actions/checkout@v2 } - - { uses: pnpm/action-setup@v2, with: { version: 7.5.2 } } + - { uses: pnpm/action-setup@v2, with: { version: "latest" } } - { name: pnpm patch, if: runner.os == 'Windows', run: ./scripts/ci/github/patch-pnpm.ps1 } - { uses: actions/setup-node@v2, with: { node-version: "${{ env.ELECTRON_MAIL_NODE_VERSION }}", cache: "pnpm" } } - { name: system setup, if: runner.os == 'Linux', run: ./scripts/ci/github/system-setup-linux.sh } @@ -60,7 +60,7 @@ jobs: steps: # TODO use YAML anchors feature when it gets supported - { uses: actions/checkout@v2 } - - { uses: pnpm/action-setup@v2, with: { version: 7.5.2 } } + - { uses: pnpm/action-setup@v2, with: { version: "latest" } } - { name: pnpm patch, if: runner.os == 'Windows', run: ./scripts/ci/github/patch-pnpm.ps1 } - { uses: actions/setup-node@v2, with: { node-version: "${{ env.ELECTRON_MAIL_NODE_VERSION }}", cache: "pnpm" } } - { name: system setup, if: runner.os == 'Linux', run: ./scripts/ci/github/system-setup-linux.sh } diff --git a/README.md b/README.md index 6bcae62ec..68aa9717f 100644 --- a/README.md +++ b/README.md @@ -78,7 +78,6 @@ The [reproducible builds](https://en.wikipedia.org/wiki/Reproducible_builds) ide - On `Linux`: `python`, `make` and a C/C++ compiler toolchain, like `GCC` are most likely already installed. Besides [keytar](https://github.com/atom/node-keytar) needs `libsecret` library to be installed. - On `macOS`: `python` and [Xcode](https://developer.apple.com/xcode/download/) need to be installed. You also need to install the `Command Line Tools` via Xcode, can be found under the `Xcode -> Preferences -> Downloads` menu. - ProtonMail's [WebClient](https://github.com/ProtonMail/WebClient) requires `yarn` to be available on your system. Additional setup is required if you run Windows, [see](https://github.com/ProtonMail/proton-shared/wiki/setup-windows). -- Make sure you have [Rust](https://www.rust-lang.org/) installed on the system. - [Clone](https://help.github.com/articles/cloning-a-repository/) this project to your local device. If you are going to contribute, consider cloning the [forked](https://help.github.com/articles/fork-a-repo/) into your own GitHub account project. - Install [pnpm](https://pnpm.io/installation). - Install dependencies running `pnpm install --frozen-lockfile` (setting `PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1` environment variable might speed up the process). diff --git a/package.json b/package.json index 4e3d52f41..212333785 100644 --- a/package.json +++ b/package.json @@ -18,6 +18,7 @@ "type": "module", "main": "./app/electron-main/index.cjs", "scripts": { + "postinstall": "node -e \"if (process.platform === 'win32') process.exit(0); require('child_process').exec('chmod +x ./node_modules/.bin/*')\"", "prepare": "npm-run-all prepare:husky-install prepare:remove:prebuild-install prepare:ts-patch prepare:ts-patch:check", "prepare:husky-install": "husky install", "prepare:ts-patch": "ts-patch install -s", @@ -124,6 +125,7 @@ "@ngrx/effects": "14.3.1", "@ngrx/store": "14.3.1", "@ngtools/webpack": "14.2.2", + "@oneidentity/zstd-js": "^1.0.2", "@playwright/test": "1.25.2", "@rgba-image/lanczos": "0.1.1", "@t-bowersox/cookie": "1.0.0", diff --git a/patches/typescript-transform-paths@3.3.1.patch b/patches/typescript-transform-paths@3.3.1.patch index cc9356d22..91d608d23 100644 --- a/patches/typescript-transform-paths@3.3.1.patch +++ b/patches/typescript-transform-paths@3.3.1.patch @@ -10,4 +10,4 @@ index 4fb647535a227ebe25fff6240f2111c1349c6596..6e2a314e1315cc5be7e6af908fae3721 + return factory.updateImportDeclaration(node, node.decorators, node.modifiers, importClause, p, node.assertClause); }); /** - * Update ExportDeclaration \ No newline at end of file + * Update ExportDeclaration diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0156d3eb9..d2ecab94d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -35,7 +35,7 @@ patchedDependencies: hash: gdagwmwndeixxlcm7duqxqq3r4 path: patches/ts-node@10.9.1.patch typescript-transform-paths@3.3.1: - hash: kk266mnbftw7tkxl4vh73rpoba + hash: izr7nv74skc2o7zxhii424xmrq path: patches/typescript-transform-paths@3.3.1.patch electron-rebuild@3.2.9: hash: 45qvjfafbmjnhciumr5bn7cpcm @@ -73,6 +73,7 @@ importers: '@ngrx/effects': 14.3.1 '@ngrx/store': 14.3.1 '@ngtools/webpack': 14.2.2 + '@oneidentity/zstd-js': ^1.0.2 '@playwright/test': 1.25.2 '@rgba-image/lanczos': 0.1.1 '@t-bowersox/cookie': 1.0.0 @@ -230,6 +231,7 @@ importers: '@ngrx/effects': 14.3.1_6eg3g2xtawfw3o6rgsxwgio6oe '@ngrx/store': 14.3.1_i7tktysnynbuwpamqlvgeyyawi '@ngtools/webpack': 14.2.2_cpi7uk4wacjg7eibn5ru6hde5u + '@oneidentity/zstd-js': 1.0.2 '@playwright/test': 1.25.2 '@rgba-image/lanczos': 0.1.1 '@t-bowersox/cookie': 1.0.0 @@ -345,7 +347,7 @@ importers: tslib: 2.4.0 turndown: 7.1.1 typescript: 4.8.3 - typescript-transform-paths: 3.3.1_kk266mnbftw7tkxl4vh73rpoba_typescript@4.8.3 + typescript-transform-paths: 3.3.1_izr7nv74skc2o7zxhii424xmrq_typescript@4.8.3 valid-url: 1.0.9 webpack: 5.74.0_webpack-cli@4.10.0 webpack-cli: 4.10.0_webpack@5.74.0 @@ -1251,6 +1253,12 @@ packages: rimraf: 3.0.2 dev: true + /@oneidentity/zstd-js/1.0.2: + resolution: {integrity: sha512-wfr61n6YIx8jJOqjbJ3CbeJ8JQo7LOHGnJuuruu10SvobuzncKxpRYaI1erQetiUn4fmN7D5m9Zj8aNR6Hl/wA==} + dependencies: + '@types/emscripten': 1.39.6 + dev: true + /@otplib/core/12.0.1: resolution: {integrity: sha512-4sGntwbA/AC+SbPhbsziRiD+jNDdIzsZ3JUyfZwjtKyc/wufl1pnSIaG4Uqx8ymPagujub0o92kgBnB89cuAMA==} dev: true @@ -1443,6 +1451,10 @@ packages: typescript: 4.8.3 dev: true + /@types/emscripten/1.39.6: + resolution: {integrity: sha512-H90aoynNhhkQP6DRweEjJp5vfUVdIj7tdPLsu7pq89vODD/lcugKfZOsfgwpvM6XUewEp2N5dCg1Uf3Qe55Dcg==} + dev: true + /@types/eslint-scope/3.7.4: resolution: {integrity: sha512-9K4zoImiZc3HlIp6AVUDE4CWYx22a+lhSZMYNpbjW04+YF0KWj4pJXnEMjdnFTiQibFFmElcsasJXDbdI/EPhA==} dependencies: @@ -7716,7 +7728,7 @@ packages: engines: {node: '>=12.20'} dev: true - /typescript-transform-paths/3.3.1_kk266mnbftw7tkxl4vh73rpoba_typescript@4.8.3: + /typescript-transform-paths/3.3.1_izr7nv74skc2o7zxhii424xmrq_typescript@4.8.3: resolution: {integrity: sha512-c+8Cqd2rsRtTU68rJI0NX/OtqgBDddNs1fIxm1nCNyhn0WpoyqtpUxc1w9Ke5c5kgE4/OT5xYbKf2cf694RYEg==} peerDependencies: typescript: '>=3.6.5' diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml deleted file mode 100644 index 175156123..000000000 --- a/pnpm-workspace.yaml +++ /dev/null @@ -1,3 +0,0 @@ -packages: - - '.' - - './src/electron-main/database/serialization/compression-native' diff --git a/scripts/electron-main-compile-compression-native.ts b/scripts/electron-main-compile-compression-native.ts deleted file mode 100644 index a462acc72..000000000 --- a/scripts/electron-main-compile-compression-native.ts +++ /dev/null @@ -1,19 +0,0 @@ -import os from "os"; - -import {catchTopLeventAsync, execShell} from "scripts/lib"; - -catchTopLeventAsync(async () => { - const rustFlagsEnvVarName = "RUSTFLAGS"; - - await execShell([ - "pnpm", - [ - ...`run --filter compression-native build`.split(" "), - ], - ...( - os.platform() === "darwin" - ? [{env: {...process.env, [rustFlagsEnvVarName]: "-C link-args=-Wl,-undefined,dynamic_lookup"}}] as const - : [] as const - ) - ], {printEnvWhitelist: [rustFlagsEnvVarName]}); -}); diff --git a/src/electron-main/context.ts b/src/electron-main/context.ts index 5fab95e7f..8fd06f75f 100644 --- a/src/electron-main/context.ts +++ b/src/electron-main/context.ts @@ -238,9 +238,9 @@ export function initContext( return {encryption}; }, }, - async dbCompression(): Promise { - const {dbCompression} = await configStore.readExisting(); - return dbCompression; + async dbCompression(): Promise { + const {dbCompression2} = await configStore.readExisting(); + return dbCompression2; }, } as const; return { diff --git a/src/electron-main/database/index.ts b/src/electron-main/database/index.ts index c916666d3..a68326fce 100644 --- a/src/electron-main/database/index.ts +++ b/src/electron-main/database/index.ts @@ -125,7 +125,7 @@ export class Database { resolveKey: () => Promise; resolvePreset: () => Promise; }>; - dbCompression: () => Promise; + dbCompression: () => Promise; }> ) { this.serializer = buildSerializer(this.options.file); diff --git a/src/electron-main/database/serialization/compression-native/.gitignore b/src/electron-main/database/serialization/compression-native/.gitignore deleted file mode 100644 index f3e9007e2..000000000 --- a/src/electron-main/database/serialization/compression-native/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -/target/ -*.node diff --git a/src/electron-main/database/serialization/compression-native/Cargo.lock b/src/electron-main/database/serialization/compression-native/Cargo.lock deleted file mode 100644 index 4df89a42b..000000000 --- a/src/electron-main/database/serialization/compression-native/Cargo.lock +++ /dev/null @@ -1,552 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 3 - -[[package]] -name = "aho-corasick" -version = "0.7.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4f55bd91a0978cbfd91c457a164bab8b4001c833b7f323132c0a4e1922dd44e" -dependencies = [ - "memchr", -] - -[[package]] -name = "async-compression" -version = "0.3.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "345fd392ab01f746c717b1357165b76f0b67a60192007b234058c9045fdcf695" -dependencies = [ - "bzip2", - "futures-core", - "memchr", - "pin-project-lite", - "tokio", - "zstd", - "zstd-safe", -] - -[[package]] -name = "autocfg" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" - -[[package]] -name = "bitflags" -version = "1.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" - -[[package]] -name = "bytes" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec8a7b6a70fde80372154c65702f00a0f56f3e1c36abbc6c440484be248856db" - -[[package]] -name = "bzip2" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6afcd980b5f3a45017c57e57a2fcccbb351cc43a356ce117ef760ef8052b89b0" -dependencies = [ - "bzip2-sys", - "libc", -] - -[[package]] -name = "bzip2-sys" -version = "0.1.11+1.0.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "736a955f3fa7875102d57c82b8cac37ec45224a07fd32d58f9f7a186b6cd4cdc" -dependencies = [ - "cc", - "libc", - "pkg-config", -] - -[[package]] -name = "cc" -version = "1.0.73" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11" -dependencies = [ - "jobserver", -] - -[[package]] -name = "cfg-if" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" - -[[package]] -name = "compression-native" -version = "0.1.0" -dependencies = [ - "async-compression", - "mimalloc-rust", - "napi", - "napi-build", - "napi-derive", - "tokio", -] - -[[package]] -name = "convert_case" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb4a24b1aaf0fd0ce8b45161144d6f42cd91677fd5940fd431183eb023b3a2b8" - -[[package]] -name = "ctor" -version = "0.1.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdffe87e1d521a10f9696f833fe502293ea446d7f256c06128293a4119bdf4cb" -dependencies = [ - "quote", - "syn", -] - -[[package]] -name = "cty" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b365fabc795046672053e29c954733ec3b05e4be654ab130fe8f1f94d7051f35" - -[[package]] -name = "futures-core" -version = "0.3.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e5aa3de05362c3fb88de6531e6296e85cde7739cccad4b9dfeeb7f6ebce56bf" - -[[package]] -name = "hermit-abi" -version = "0.1.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" -dependencies = [ - "libc", -] - -[[package]] -name = "jobserver" -version = "0.1.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af25a77299a7f711a01975c35a6a424eb6862092cc2d6c72c4ed6cbc56dfc1fa" -dependencies = [ - "libc", -] - -[[package]] -name = "libc" -version = "0.2.133" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0f80d65747a3e43d1596c7c5492d95d5edddaabd45a7fcdb02b95f644164966" - -[[package]] -name = "libloading" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efbc0f03f9a775e9f6aed295c6a1ba2253c5757a9e03d55c6caa46a681abcddd" -dependencies = [ - "cfg-if", - "winapi", -] - -[[package]] -name = "lock_api" -version = "0.4.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df" -dependencies = [ - "autocfg", - "scopeguard", -] - -[[package]] -name = "log" -version = "0.4.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "memchr" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" - -[[package]] -name = "mimalloc-rust" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcc30df9dfdb5bb6cb2470de65ca604c3eaa3e5dc2ad02a9a98f567df5844472" -dependencies = [ - "cty", - "mimalloc-rust-sys", -] - -[[package]] -name = "mimalloc-rust-sys" -version = "1.7.3-source" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3adc8731262b982f4e0860770dba118305cafe1b2e7ebe95b29b2c2f46a70666" -dependencies = [ - "cc", - "cty", -] - -[[package]] -name = "mio" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57ee1c23c7c63b0c9250c339ffdc69255f110b298b901b9f6c82547b7b87caaf" -dependencies = [ - "libc", - "log", - "wasi", - "windows-sys", -] - -[[package]] -name = "napi" -version = "2.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "743fece4c26c5132f8559080145fde9ba88700c0f1aa30a1ab3e057ab105814d" -dependencies = [ - "bitflags", - "ctor", - "napi-sys", - "once_cell", - "thread_local", - "tokio", -] - -[[package]] -name = "napi-build" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "882a73d9ef23e8dc2ebbffb6a6ae2ef467c0f18ac10711e4cc59c5485d41df0e" - -[[package]] -name = "napi-derive" -version = "2.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39f3d8b02ef355898ea98f69082d9a183c8701c836942c2daf3e92364e88a0fa" -dependencies = [ - "convert_case", - "napi-derive-backend", - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "napi-derive-backend" -version = "1.0.38" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c35640513eb442fcbd1653a1c112fb6b2cc12b54d82f9c141f5859c721cab36" -dependencies = [ - "convert_case", - "once_cell", - "proc-macro2", - "quote", - "regex", - "syn", -] - -[[package]] -name = "napi-sys" -version = "2.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "529671ebfae679f2ce9630b62dd53c72c56b3eb8b2c852e7e2fa91704ff93d67" -dependencies = [ - "libloading", -] - -[[package]] -name = "num_cpus" -version = "1.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19e64526ebdee182341572e50e9ad03965aa510cd94427a4549448f285e957a1" -dependencies = [ - "hermit-abi", - "libc", -] - -[[package]] -name = "once_cell" -version = "1.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e82dad04139b71a90c080c8463fe0dc7902db5192d939bd0950f074d014339e1" - -[[package]] -name = "parking_lot" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" -dependencies = [ - "lock_api", - "parking_lot_core", -] - -[[package]] -name = "parking_lot_core" -version = "0.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09a279cbf25cb0757810394fbc1e359949b59e348145c643a939a525692e6929" -dependencies = [ - "cfg-if", - "libc", - "redox_syscall", - "smallvec", - "windows-sys", -] - -[[package]] -name = "pin-project-lite" -version = "0.2.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" - -[[package]] -name = "pkg-config" -version = "0.3.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1df8c4ec4b0627e53bdf214615ad287367e482558cf84b109250b37464dc03ae" - -[[package]] -name = "proc-macro2" -version = "1.0.43" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a2ca2c61bc9f3d74d2886294ab7b9853abd9c1ad903a3ac7815c58989bb7bab" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "quote" -version = "1.0.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbe448f377a7d6961e30f5955f9b8d106c3f5e449d493ee1b125c1d43c2b5179" -dependencies = [ - "proc-macro2", -] - -[[package]] -name = "redox_syscall" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" -dependencies = [ - "bitflags", -] - -[[package]] -name = "regex" -version = "1.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c4eb3267174b8c6c2f654116623910a0fef09c4753f8dd83db29c48a0df988b" -dependencies = [ - "aho-corasick", - "memchr", - "regex-syntax", -] - -[[package]] -name = "regex-syntax" -version = "0.6.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3f87b73ce11b1619a3c6332f45341e0047173771e8b8b73f87bfeefb7b56244" - -[[package]] -name = "scopeguard" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" - -[[package]] -name = "signal-hook-registry" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51e73328dc4ac0c7ccbda3a494dfa03df1de2f46018127f60c693f2648455b0" -dependencies = [ - "libc", -] - -[[package]] -name = "smallvec" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fd0db749597d91ff862fd1d55ea87f7855a744a8425a64695b6fca237d1dad1" - -[[package]] -name = "socket2" -version = "0.4.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02e2d2db9033d13a1567121ddd7a095ee144db4e1ca1b1bda3419bc0da294ebd" -dependencies = [ - "libc", - "winapi", -] - -[[package]] -name = "syn" -version = "1.0.100" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52205623b1b0f064a4e71182c3b18ae902267282930c6d5462c91b859668426e" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "thread_local" -version = "1.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5516c27b78311c50bf42c071425c560ac799b11c30b31f87e3081965fe5e0180" -dependencies = [ - "once_cell", -] - -[[package]] -name = "tokio" -version = "1.21.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0020c875007ad96677dcc890298f4b942882c5d4eb7cc8f439fc3bf813dc9c95" -dependencies = [ - "autocfg", - "bytes", - "libc", - "memchr", - "mio", - "num_cpus", - "once_cell", - "parking_lot", - "pin-project-lite", - "signal-hook-registry", - "socket2", - "tokio-macros", - "winapi", -] - -[[package]] -name = "tokio-macros" -version = "1.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9724f9a975fb987ef7a3cd9be0350edcbe130698af5b8f7a631e23d42d052484" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "unicode-ident" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcc811dc4066ac62f84f11307873c4850cb653bfa9b1719cee2bd2204a4bc5dd" - -[[package]] -name = "wasi" -version = "0.11.0+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" - -[[package]] -name = "winapi" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" -dependencies = [ - "winapi-i686-pc-windows-gnu", - "winapi-x86_64-pc-windows-gnu", -] - -[[package]] -name = "winapi-i686-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" - -[[package]] -name = "winapi-x86_64-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" - -[[package]] -name = "windows-sys" -version = "0.36.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2" -dependencies = [ - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_msvc", -] - -[[package]] -name = "windows_aarch64_msvc" -version = "0.36.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47" - -[[package]] -name = "windows_i686_gnu" -version = "0.36.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6" - -[[package]] -name = "windows_i686_msvc" -version = "0.36.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.36.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.36.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680" - -[[package]] -name = "zstd" -version = "0.11.2+zstd.1.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4" -dependencies = [ - "zstd-safe", -] - -[[package]] -name = "zstd-safe" -version = "5.0.2+zstd.1.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d2a5585e04f9eea4b2a3d1eca508c4dee9592a89ef6f450c11719da0726f4db" -dependencies = [ - "libc", - "zstd-sys", -] - -[[package]] -name = "zstd-sys" -version = "2.0.1+zstd.1.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fd07cbbc53846d9145dbffdf6dd09a7a0aa52be46741825f5c97bdd4f73f12b" -dependencies = [ - "cc", - "libc", -] diff --git a/src/electron-main/database/serialization/compression-native/Cargo.toml b/src/electron-main/database/serialization/compression-native/Cargo.toml deleted file mode 100644 index d746c7822..000000000 --- a/src/electron-main/database/serialization/compression-native/Cargo.toml +++ /dev/null @@ -1,22 +0,0 @@ -[package] -name = "compression-native" -version = "0.1.0" -edition = "2021" - -[lib] -crate-type = ["cdylib"] - -[dependencies] -napi = { version = "2", features = ["napi8", "tokio_rt"] } -napi-derive = "2" -async-compression = { version = "0.3", features = ["tokio", "zstd", "bzip2"] } -tokio = { version = "1", features = ["full"] } - -[target.'cfg(all(target_arch = "x86_64", not(target_env = "musl")))'.dependencies] -mimalloc-rust = "0.1" - -[build-dependencies] -napi-build = "2" - -[profile.release] -lto = true diff --git a/src/electron-main/database/serialization/compression-native/index.d.ts b/src/electron-main/database/serialization/compression-native/index.d.ts deleted file mode 100644 index a49d3fa73..000000000 --- a/src/electron-main/database/serialization/compression-native/index.d.ts +++ /dev/null @@ -1,7 +0,0 @@ -/* tslint:disable */ -/* eslint-disable */ - -/* auto-generated by NAPI-RS */ - -export function decompress(compressionType: 'bzip2' | 'zstd', input: Buffer): Promise -export function compress(compressionType: 'bzip2' | 'zstd', input: Buffer, level: number): Promise diff --git a/src/electron-main/database/serialization/compression-native/package.json b/src/electron-main/database/serialization/compression-native/package.json deleted file mode 100644 index b3ba34ffa..000000000 --- a/src/electron-main/database/serialization/compression-native/package.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "name": "compression-native", - "version": "0.1.0", - "main": "index.node", - "scripts": { - "build": "napi build --release --strip" - }, - "devDependencies": { - "@napi-rs/cli": "^2" - } -} diff --git a/src/electron-main/database/serialization/compression-native/src/lib.rs b/src/electron-main/database/serialization/compression-native/src/lib.rs deleted file mode 100644 index b4757c120..000000000 --- a/src/electron-main/database/serialization/compression-native/src/lib.rs +++ /dev/null @@ -1,47 +0,0 @@ -#![allow(non_camel_case_types, non_snake_case)] - -use std::io::Cursor; - -use async_compression::{Level, tokio::bufread::BzDecoder, tokio::bufread::ZstdDecoder, tokio::write::BzEncoder, tokio::write::ZstdEncoder}; -use napi::bindgen_prelude::{Buffer, Error, Result, Status}; -use napi_derive::napi; -use tokio::io::{AsyncReadExt, AsyncWriteExt}; - -macro_rules! performAction { - ($decoder: path, $input: expr) => { - { - let input: &[u8] = $input.as_ref(); - let mut output = Vec::new(); - let mut decoder = $decoder(Cursor::new(input)); - decoder.read_to_end(&mut output).await?; - Ok(Buffer::from(output)) - } - }; - ($encoder: path, $input: expr, $level: expr) => { - { - let input: &[u8] = $input.as_ref(); - let mut encoder = $encoder(Vec::new(), Level::Precise($level)); - encoder.write_all(input).await?; - encoder.shutdown().await?; - Ok(Buffer::from(encoder.into_inner())) - } - }; -} - -#[napi(ts_args_type = "compressionType: 'bzip2' | 'zstd', input: Buffer")] -pub async fn decompress(compressionType: String, input: Buffer) -> Result { - match compressionType.as_str() { - "bzip2" => performAction!(BzDecoder::new, input), - "zstd" => performAction!(ZstdDecoder::new, input), - _ => Err(Error::new(Status::InvalidArg, "Compression type".to_owned())), - } -} - -#[napi(ts_args_type = "compressionType: 'bzip2' | 'zstd', input: Buffer, level: number")] -pub async fn compress(compressionType: String, input: Buffer, level: u32) -> Result { - match compressionType.as_str() { - "bzip2" => performAction!(BzEncoder::with_quality, input, level), - "zstd" => performAction!(ZstdEncoder::with_quality, input, level), - _ => Err(Error::new(Status::InvalidArg, "Compression type".to_owned())), - } -} diff --git a/src/electron-main/database/serialization/index.ts b/src/electron-main/database/serialization/index.ts index bb1daeecf..88d522121 100644 --- a/src/electron-main/database/serialization/index.ts +++ b/src/electron-main/database/serialization/index.ts @@ -27,6 +27,18 @@ const fsAsync = { // TODO use "fs/promises" (at the moment there is no "read" fu const msgpackr = new Packr({useRecords: false, moreTypes: false, structuredClone: false, int64AsNumber: true}); +export const resolveZstdWasm = ((): () => Promise => { + let lib: typeof import("@oneidentity/zstd-js/wasm")["ZstdSimple"] | undefined; + return async () => { + if (!lib) { + const {ZstdInit} = await import("@oneidentity/zstd-js/wasm"); + const {ZstdSimple} = await ZstdInit(); + lib = ZstdSimple; + } + return lib; + }; +})(); + export const buildSerializer: Model.buildSerializerType = (() => { const CONST = { zero: 0, @@ -37,41 +49,35 @@ export const buildSerializer: Model.buildSerializerType = (() => { dataReadingBufferSize: ONE_MB_BYTES, dataWritingBufferSize: ONE_MB_BYTES, mailsPortionSize: {min: 400, max: 5000}, - compressionLevelsFallback: {gzip: 6, zstd: 7, bzip2: 1}, + compressionLevelsFallback: {gzip: 6, zstd: 7}, } as const; const decryptBuffer = async ( data: Buffer, encryptionAdapter: EncryptionAdapter, - compressionType?: Config["dbCompression"]["type"], + compressionType?: Config["dbCompression2"]["type"], ): Promise => { const decryptedBuffer = await encryptionAdapter.read(data); return compressionType === "gzip" ? promisify(zlib.gunzip)(decryptedBuffer) - : (compressionType === "zstd" || compressionType === "bzip2") - ? (await import("./compression-native")).decompress(compressionType, decryptedBuffer) + : (compressionType === "zstd") + ? Buffer.from( + (await resolveZstdWasm()).decompress(decryptedBuffer), + ) : decryptedBuffer; }; const serializeDataMapItem = async ( data: DeepReadonly | DeepReadonly, encryptionAdapter: EncryptionAdapter, - compressionType: Config["dbCompression"]["type"], - level: Config["dbCompression"]["level"], + compressionType: Config["dbCompression2"]["type"], + level: Config["dbCompression2"]["level"], ): Promise => { if (typeof level !== "number" || level < 1 || - ( - compressionType === "gzip" && level > 9 - ) - || - ( - compressionType === "bzip2" && level > 9 - ) + (compressionType === "gzip" && level > 9) || - ( - compressionType === "zstd" && level > 22 - ) + (compressionType === "zstd" && level > 22) ) { level = CONST.compressionLevelsFallback[compressionType]; logger.error(`Invalid "${nameof(level)}" value, falling back to the default value: ${level}`); @@ -81,8 +87,13 @@ export const buildSerializer: Model.buildSerializerType = (() => { const encryptedData = await encryptionAdapter.write( compressionType === "gzip" ? await promisify(zlib.gzip)(serializedDataBuffer, {level}) - : (compressionType === "zstd" || compressionType === "bzip2") - ? await (await import("./compression-native")).compress(compressionType, serializedDataBuffer, level) + // TODO explore the "zstd" library payload size limitation: min 100 bytes + // eslint-disable-next-line max-len + // https://github.com/OneIdentity/zstd-js/blob/d9de2d24e6b61ae9d3cf86c2838a117555c1e835/src/components/common/zstd-simple/zstd-simple.ts#L23 + : (compressionType === "zstd" && serializedDataBuffer.byteLength > 100) + ? Buffer.from( + (await resolveZstdWasm()).compress(serializedDataBuffer, level), + ) : serializedDataBuffer ); const encryptionHeaderBuffer = encryptedData.slice(CONST.zero, encryptedData.indexOf(CONST.headerZeroByteMark)); diff --git a/src/electron-main/database/serialization/model.ts b/src/electron-main/database/serialization/model.ts index 360a5c54c..914799bb6 100644 --- a/src/electron-main/database/serialization/model.ts +++ b/src/electron-main/database/serialization/model.ts @@ -6,7 +6,7 @@ import type {FsDb} from "src/shared/model/database"; export type DataMapSerializationHeaderPart = DeepReadonly<{ type: "msgpack" dataMap?: { - readonly compression?: Config["dbCompression"]["type"] + readonly compression?: Config["dbCompression2"]["type"] readonly items: Array<{ byteLength: number }> } }>; @@ -24,6 +24,6 @@ export type buildSerializerType = (file: string) => { write: ( encryptionAdapter: EncryptionAdapter, data: DeepReadonly, - dbCompression: DeepReadonly, + dbCompression: DeepReadonly, ) => Promise }; diff --git a/src/electron-main/storage-upgrade.ts b/src/electron-main/storage-upgrade.ts index 6af47cc97..826a624ed 100644 --- a/src/electron-main/storage-upgrade.ts +++ b/src/electron-main/storage-upgrade.ts @@ -339,14 +339,6 @@ const CONFIG_UPGRADES: Record void> = { } } }, - "4.15.1": (config) => { - { - const key = "dbCompression"; - if (typeof config[key] === "undefined") { - config[key] = INITIAL_STORES.config()[key]; - } - } - }, "5.1.0": (config) => { { const key = "spellcheck"; @@ -382,6 +374,14 @@ const CONFIG_UPGRADES: Record void> = { delete (config.timeouts as { dbBootstrapping?: unknown }).dbBootstrapping; delete (config.timeouts as { dbSyncing?: unknown }).dbSyncing; }, + "5.1.2": (config) => { + { + const key = "dbCompression2"; + if (typeof config[key] === "undefined") { + config[key] = INITIAL_STORES.config()[key]; + } + } + }, // last updater "100.0.0": (config) => { // ensuring default base props are set diff --git a/src/shared/model/options.ts b/src/shared/model/options.ts index ee1fae068..b08f52ef6 100644 --- a/src/shared/model/options.ts +++ b/src/shared/model/options.ts @@ -48,7 +48,7 @@ export interface Config extends BaseConfig, Partial { dbSyncingFiredTriggerDebounce: number; shouldRequestDbMetadataReset: "initial" | "done"; // TODO consider making compression type/level configurable via interface - dbCompression: { type: "gzip" | "zstd" | "bzip2", level: number, mailsPortionSize: { min: number, max: number } }; + dbCompression2: { type: "gzip" | "zstd", level: number, mailsPortionSize: { min: number, max: number } }; dbMergeBytesFileSizeThreshold: number // base calendarNotification: boolean diff --git a/src/shared/util/index.ts b/src/shared/util/index.ts index ef2c1fce2..21195b2bb 100644 --- a/src/shared/util/index.ts +++ b/src/shared/util/index.ts @@ -70,11 +70,7 @@ export function initialConfig(): Config { dbSyncingOnlineTriggerDelay: ONE_SECOND_MS * 3, dbSyncingFiredTriggerDebounce: ONE_SECOND_MS * 5, shouldRequestDbMetadataReset: "initial", - dbCompression: { - type: BUILD_ENVIRONMENT === "e2e" ? "zstd" : "gzip", - level: 7, - mailsPortionSize: {min: 800, max: 1000}, - }, + dbCompression2: {type: "zstd", level: 3, mailsPortionSize: {min: 700, max: 900}}, dbMergeBytesFileSizeThreshold: 1024 * 1024 * 5, // 5 MB // base calendarNotification: false, diff --git a/webpack-configs/electron-main.ts b/webpack-configs/electron-main.ts index 82e51986e..32019eb57 100644 --- a/webpack-configs/electron-main.ts +++ b/webpack-configs/electron-main.ts @@ -18,10 +18,6 @@ export default buildBaseConfig( module: { rules: [ typescriptLoaderRule({tsConfigFile}), - { - test: /\.node$/, - loader: "node-loader", - }, ], }, externals: mapValues(