From 68aa6953d4e9dc159270c4236432f4d52a71a308 Mon Sep 17 00:00:00 2001 From: darkobits Date: Wed, 15 Jan 2020 14:31:13 -0800 Subject: [PATCH] refactor: FetchStickerDataPlugin caches sticker data. --- .gitignore | 5 +- config/webpack.config.ts | 10 +- package-lock.json | 302 ++++++++++++++++-- package.json | 8 +- src/components/contribute/Contribute.tsx | 4 +- src/components/home/Home.tsx | 2 +- src/components/home/SearchResults.tsx | 4 +- .../home/StickerPackPreviewCard.tsx | 6 +- src/components/layout/Footer.tsx | 2 +- src/components/pack/Sticker.tsx | 4 +- src/components/pack/StickerPackDetail.tsx | 83 ++--- src/contexts/StickersContext.tsx | 4 +- src/etc/Stickers.proto | 12 - src/etc/stickers-proto.ts | 40 +++ src/etc/types.ts | 38 ++- src/lib/{convert.ts => convert-image.ts} | 7 +- src/lib/crypto.ts | 114 ++++--- src/lib/signal.ts | 134 ++++++++ src/lib/stickers.ts | 183 +++++------ src/plugins/FetchStickerDataPlugin.ts | 210 ++++++------ 20 files changed, 779 insertions(+), 393 deletions(-) delete mode 100644 src/etc/Stickers.proto create mode 100644 src/etc/stickers-proto.ts rename src/lib/{convert.ts => convert-image.ts} (95%) create mode 100644 src/lib/signal.ts diff --git a/.gitignore b/.gitignore index b04afad1..f3721640 100644 --- a/.gitignore +++ b/.gitignore @@ -87,10 +87,11 @@ typings/ # next.js build output .next -# Build artifacts +# Build artifacts. /dist # Linaria's cache directory. /.linaria-cache -/NOTES.md +# Cache directory for FetchStickerDataPlugin. +/.sticker-pack-cache diff --git a/config/webpack.config.ts b/config/webpack.config.ts index f6fdfc2d..55372c1b 100644 --- a/config/webpack.config.ts +++ b/config/webpack.config.ts @@ -105,9 +105,9 @@ export default (env: string, argv: any): webpack.Configuration => { }] }); - // Text & Protobuf files. + // Text files. config.module.rules.push({ - test: /\.(txt|proto)$/, + test: /\.txt$/, use: [{ loader: 'raw-loader' }] @@ -212,9 +212,11 @@ export default (env: string, argv: any): webpack.Configuration => { } config.plugins.push(new FetchStickerDataPlugin({ - filename: 'stickerData.json' + inputFile: path.resolve(PKG_ROOT, 'stickers.yml'), + outputFile: 'stickerData.json' })); + // ----- Dev Server ---------------------------------------------------------- if (argv.mode === 'development') { @@ -256,7 +258,7 @@ export default (env: string, argv: any): webpack.Configuration => { minChunks: 1 }, data: { - test: /src\/.(json|proto)$/, + test: /src\/.json$/, name: 'data', chunks: 'all', minChunks: 1 diff --git a/package-lock.json b/package-lock.json index bbe6fd20..9f77b2e9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -973,9 +973,9 @@ } }, "@babel/register": { - "version": "7.7.7", - "resolved": "https://registry.npmjs.org/@babel/register/-/register-7.7.7.tgz", - "integrity": "sha512-S2mv9a5dc2pcpg/ConlKZx/6wXaEwHeqfo7x/QbXsdCAZm+WJC1ekVvL1TVxNsedTs5y/gG63MhJTEsmwmjtiA==", + "version": "7.8.0", + "resolved": "https://registry.npmjs.org/@babel/register/-/register-7.8.0.tgz", + "integrity": "sha512-i7CPQBzb/uALrZZozm6jXpSoieZvcTSOqonKA9UX4OLEvAYc4Y2VqgW67ZkSz6xfaNP6m1g1oBy0/zMA7YcdJA==", "requires": { "find-cache-dir": "^2.0.0", "lodash": "^4.17.13", @@ -2335,6 +2335,15 @@ "@types/webpack": "*" } }, + "@types/fs-extra": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-8.0.1.tgz", + "integrity": "sha512-J00cVDALmi/hJOYsunyT52Hva5TnJeKP5yd1r+mH/ZU0mbYZflR0Z5kw5kITtKTRYMhm1JMClOFYdHnQszEvqw==", + "dev": true, + "requires": { + "@types/node": "*" + } + }, "@types/fuzzy-search": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/@types/fuzzy-search/-/fuzzy-search-2.1.0.tgz", @@ -2434,6 +2443,12 @@ "jest-diff": "^24.3.0" } }, + "@types/js-yaml": { + "version": "3.12.1", + "resolved": "https://registry.npmjs.org/@types/js-yaml/-/js-yaml-3.12.1.tgz", + "integrity": "sha512-SGGAhXLHDx+PK4YLNcNGa6goPf9XRWQNAUUbffkwVGGXIxmDKWyGGL4inzq2sPmExu431Ekb9aEMn9BkPqEYFA==", + "dev": true + }, "@types/long": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/@types/long/-/long-4.0.0.tgz", @@ -2461,9 +2476,9 @@ "dev": true }, "@types/node": { - "version": "13.1.4", - "resolved": "https://registry.npmjs.org/@types/node/-/node-13.1.4.tgz", - "integrity": "sha512-Lue/mlp2egZJoHXZr4LndxDAd7i/7SQYhV0EjWfb/a4/OZ6tuVwMCVPiwkU5nsEipxEf7hmkSU7Em5VQ8P5NGA==", + "version": "13.1.7", + "resolved": "https://registry.npmjs.org/@types/node/-/node-13.1.7.tgz", + "integrity": "sha512-HU0q9GXazqiKwviVxg9SI/+t/nAsGkvLDkIdxz+ObejG2nX6Si00TeLqHMoS+a/1tjH7a8YpKVQwtgHuMQsldg==", "dev": true }, "@types/p-wait-for": { @@ -2475,6 +2490,15 @@ "p-wait-for": "*" } }, + "@types/progress": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@types/progress/-/progress-2.0.3.tgz", + "integrity": "sha512-bPOsfCZ4tsTlKiBjBhKnM8jpY5nmIll166IPD58D92hR7G7kZDfx5iB9wGF4NfZrdKolebjeAr3GouYkSGoJ/A==", + "dev": true, + "requires": { + "@types/node": "*" + } + }, "@types/prop-types": { "version": "15.7.3", "resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.3.tgz", @@ -3363,6 +3387,36 @@ "find-up": "^3.0.0", "istanbul-lib-instrument": "^3.3.0", "test-exclude": "^5.2.3" + }, + "dependencies": { + "find-up": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-3.0.0.tgz", + "integrity": "sha512-1yD6RmLI1XBfxugvORwlck6f75tYL+iR0jqwsOrOxMZyGYqUuDhJ0l4AXdO1iX/FTs9cBAMEk1gWSEx1kSbylg==", + "dev": true, + "requires": { + "locate-path": "^3.0.0" + } + }, + "locate-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-3.0.0.tgz", + "integrity": "sha512-7AO748wWnIhNqAuaty2ZWHkQHRSNfPVIsPIfwEOWO22AmaoVrWavlOcMR5nzTLNYvp36X220/maaRsrec1G65A==", + "dev": true, + "requires": { + "p-locate": "^3.0.0", + "path-exists": "^3.0.0" + } + }, + "p-locate": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-3.0.0.tgz", + "integrity": "sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ==", + "dev": true, + "requires": { + "p-limit": "^2.0.0" + } + } } }, "babel-plugin-jest-hoist": { @@ -6101,11 +6155,21 @@ "dev": true }, "find-up": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/find-up/-/find-up-3.0.0.tgz", - "integrity": "sha512-1yD6RmLI1XBfxugvORwlck6f75tYL+iR0jqwsOrOxMZyGYqUuDhJ0l4AXdO1iX/FTs9cBAMEk1gWSEx1kSbylg==", + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", + "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", + "dev": true, "requires": { - "locate-path": "^3.0.0" + "locate-path": "^5.0.0", + "path-exists": "^4.0.0" + }, + "dependencies": { + "path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", + "dev": true + } } }, "findup-sync": { @@ -6416,12 +6480,12 @@ "dev": true }, "fs-extra": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-7.0.1.tgz", - "integrity": "sha512-YJDaCJZEnBmcbw13fvdAM9AwNOJwOzrE4pqMqBq5nFiEqXUqHwlK4B+3pUw6JNvfSPtX05xFHtYy/1ni01eGCw==", + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-8.1.0.tgz", + "integrity": "sha512-yhlQgA6mnOJUKOsRUFsgJdQCvkKhcz8tlZG5HBQfReYZy46OwLcY+Zia0mtdHsOo9y/hP+CxMN0TU9QxoOtG4g==", "dev": true, "requires": { - "graceful-fs": "^4.1.2", + "graceful-fs": "^4.2.0", "jsonfile": "^4.0.0", "universalify": "^0.1.0" } @@ -7126,6 +7190,17 @@ "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==", "dev": true }, + "fs-extra": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-7.0.1.tgz", + "integrity": "sha512-YJDaCJZEnBmcbw13fvdAM9AwNOJwOzrE4pqMqBq5nFiEqXUqHwlK4B+3pUw6JNvfSPtX05xFHtYy/1ni01eGCw==", + "dev": true, + "requires": { + "graceful-fs": "^4.1.2", + "jsonfile": "^4.0.0", + "universalify": "^0.1.0" + } + }, "globby": { "version": "6.1.0", "resolved": "https://registry.npmjs.org/globby/-/globby-6.1.0.tgz", @@ -9554,12 +9629,12 @@ } }, "locate-path": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-3.0.0.tgz", - "integrity": "sha512-7AO748wWnIhNqAuaty2ZWHkQHRSNfPVIsPIfwEOWO22AmaoVrWavlOcMR5nzTLNYvp36X220/maaRsrec1G65A==", + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", + "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", + "dev": true, "requires": { - "p-locate": "^3.0.0", - "path-exists": "^3.0.0" + "p-locate": "^4.1.0" } }, "lodash": { @@ -10560,11 +10635,12 @@ } }, "p-locate": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-3.0.0.tgz", - "integrity": "sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ==", + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", + "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", + "dev": true, "requires": { - "p-limit": "^2.0.0" + "p-limit": "^2.2.0" } }, "p-map": { @@ -10891,6 +10967,33 @@ "integrity": "sha512-/E57AYkoeQ25qkxMj5PBOVgF8Kiu/h7cYS30Z5+R7WaiCCBfLq58ZI/dSeaEKb9WVJV5n/03QwrN3IeWIFllvw==", "requires": { "find-up": "^3.0.0" + }, + "dependencies": { + "find-up": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-3.0.0.tgz", + "integrity": "sha512-1yD6RmLI1XBfxugvORwlck6f75tYL+iR0jqwsOrOxMZyGYqUuDhJ0l4AXdO1iX/FTs9cBAMEk1gWSEx1kSbylg==", + "requires": { + "locate-path": "^3.0.0" + } + }, + "locate-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-3.0.0.tgz", + "integrity": "sha512-7AO748wWnIhNqAuaty2ZWHkQHRSNfPVIsPIfwEOWO22AmaoVrWavlOcMR5nzTLNYvp36X220/maaRsrec1G65A==", + "requires": { + "p-locate": "^3.0.0", + "path-exists": "^3.0.0" + } + }, + "p-locate": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-3.0.0.tgz", + "integrity": "sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ==", + "requires": { + "p-limit": "^2.0.0" + } + } } }, "pkg-up": { @@ -10900,6 +11003,36 @@ "dev": true, "requires": { "find-up": "^3.0.0" + }, + "dependencies": { + "find-up": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-3.0.0.tgz", + "integrity": "sha512-1yD6RmLI1XBfxugvORwlck6f75tYL+iR0jqwsOrOxMZyGYqUuDhJ0l4AXdO1iX/FTs9cBAMEk1gWSEx1kSbylg==", + "dev": true, + "requires": { + "locate-path": "^3.0.0" + } + }, + "locate-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-3.0.0.tgz", + "integrity": "sha512-7AO748wWnIhNqAuaty2ZWHkQHRSNfPVIsPIfwEOWO22AmaoVrWavlOcMR5nzTLNYvp36X220/maaRsrec1G65A==", + "dev": true, + "requires": { + "p-locate": "^3.0.0", + "path-exists": "^3.0.0" + } + }, + "p-locate": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-3.0.0.tgz", + "integrity": "sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ==", + "dev": true, + "requires": { + "p-limit": "^2.0.0" + } + } } }, "pn": { @@ -11113,6 +11246,12 @@ "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" }, + "progress": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz", + "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==", + "dev": true + }, "promise-inflight": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/promise-inflight/-/promise-inflight-1.0.1.tgz", @@ -13375,6 +13514,25 @@ "require-main-filename": "^2.0.0" }, "dependencies": { + "locate-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-3.0.0.tgz", + "integrity": "sha512-7AO748wWnIhNqAuaty2ZWHkQHRSNfPVIsPIfwEOWO22AmaoVrWavlOcMR5nzTLNYvp36X220/maaRsrec1G65A==", + "dev": true, + "requires": { + "p-locate": "^3.0.0", + "path-exists": "^3.0.0" + } + }, + "p-locate": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-3.0.0.tgz", + "integrity": "sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ==", + "dev": true, + "requires": { + "p-limit": "^2.0.0" + } + }, "read-pkg-up": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/read-pkg-up/-/read-pkg-up-4.0.0.tgz", @@ -13383,6 +13541,17 @@ "requires": { "find-up": "^3.0.0", "read-pkg": "^3.0.0" + }, + "dependencies": { + "find-up": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-3.0.0.tgz", + "integrity": "sha512-1yD6RmLI1XBfxugvORwlck6f75tYL+iR0jqwsOrOxMZyGYqUuDhJ0l4AXdO1iX/FTs9cBAMEk1gWSEx1kSbylg==", + "dev": true, + "requires": { + "locate-path": "^3.0.0" + } + } } } } @@ -14427,6 +14596,16 @@ "tapable": "^1.0.0" } }, + "locate-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-3.0.0.tgz", + "integrity": "sha512-7AO748wWnIhNqAuaty2ZWHkQHRSNfPVIsPIfwEOWO22AmaoVrWavlOcMR5nzTLNYvp36X220/maaRsrec1G65A==", + "dev": true, + "requires": { + "p-locate": "^3.0.0", + "path-exists": "^3.0.0" + } + }, "memory-fs": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/memory-fs/-/memory-fs-0.4.1.tgz", @@ -14437,6 +14616,15 @@ "readable-stream": "^2.0.1" } }, + "p-locate": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-3.0.0.tgz", + "integrity": "sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ==", + "dev": true, + "requires": { + "p-limit": "^2.0.0" + } + }, "supports-color": { "version": "6.1.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-6.1.0.tgz", @@ -14463,6 +14651,17 @@ "which-module": "^2.0.0", "y18n": "^4.0.0", "yargs-parser": "^13.1.0" + }, + "dependencies": { + "find-up": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-3.0.0.tgz", + "integrity": "sha512-1yD6RmLI1XBfxugvORwlck6f75tYL+iR0jqwsOrOxMZyGYqUuDhJ0l4AXdO1iX/FTs9cBAMEk1gWSEx1kSbylg==", + "dev": true, + "requires": { + "locate-path": "^3.0.0" + } + } } } } @@ -14624,12 +14823,31 @@ } } }, + "locate-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-3.0.0.tgz", + "integrity": "sha512-7AO748wWnIhNqAuaty2ZWHkQHRSNfPVIsPIfwEOWO22AmaoVrWavlOcMR5nzTLNYvp36X220/maaRsrec1G65A==", + "dev": true, + "requires": { + "p-locate": "^3.0.0", + "path-exists": "^3.0.0" + } + }, "ms": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", "dev": true }, + "p-locate": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-3.0.0.tgz", + "integrity": "sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ==", + "dev": true, + "requires": { + "p-limit": "^2.0.0" + } + }, "p-map": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/p-map/-/p-map-2.1.0.tgz", @@ -14743,6 +14961,17 @@ "which-module": "^2.0.0", "y18n": "^3.2.1 || ^4.0.0", "yargs-parser": "^11.1.1" + }, + "dependencies": { + "find-up": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-3.0.0.tgz", + "integrity": "sha512-1yD6RmLI1XBfxugvORwlck6f75tYL+iR0jqwsOrOxMZyGYqUuDhJ0l4AXdO1iX/FTs9cBAMEk1gWSEx1kSbylg==", + "dev": true, + "requires": { + "locate-path": "^3.0.0" + } + } } }, "yargs-parser": { @@ -15012,6 +15241,33 @@ "which-module": "^2.0.0", "y18n": "^4.0.0", "yargs-parser": "^13.1.1" + }, + "dependencies": { + "find-up": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-3.0.0.tgz", + "integrity": "sha512-1yD6RmLI1XBfxugvORwlck6f75tYL+iR0jqwsOrOxMZyGYqUuDhJ0l4AXdO1iX/FTs9cBAMEk1gWSEx1kSbylg==", + "requires": { + "locate-path": "^3.0.0" + } + }, + "locate-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-3.0.0.tgz", + "integrity": "sha512-7AO748wWnIhNqAuaty2ZWHkQHRSNfPVIsPIfwEOWO22AmaoVrWavlOcMR5nzTLNYvp36X220/maaRsrec1G65A==", + "requires": { + "p-locate": "^3.0.0", + "path-exists": "^3.0.0" + } + }, + "p-locate": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-3.0.0.tgz", + "integrity": "sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ==", + "requires": { + "p-limit": "^2.0.0" + } + } } }, "yargs-parser": { diff --git a/package.json b/package.json index 25ae36c5..2d3edb32 100644 --- a/package.json +++ b/package.json @@ -62,12 +62,15 @@ "@types/bytes": "^3.1.0", "@types/copy-webpack-plugin": "^5.0.0", "@types/friendly-errors-webpack-plugin": "^0.1.1", + "@types/fs-extra": "^8.0.1", "@types/fuzzy-search": "^2.1.0", "@types/html-webpack-plugin": "^3.2.0", "@types/jest": "^24.0.25", + "@types/js-yaml": "^3.12.1", "@types/mini-css-extract-plugin": "^0.9.0", - "@types/node": "^13.1.4", + "@types/node": "^13.1.7", "@types/p-wait-for": "^3.0.1", + "@types/progress": "^2.0.3", "@types/ramda": "^0.26.39", "@types/react": "^16.9.13", "@types/react-dom": "^16.9.4", @@ -83,7 +86,9 @@ "del-cli": "^3.0.0", "favicons-webpack-plugin": "^2.1.0", "file-loader": "^5.0.2", + "find-up": "^4.1.0", "friendly-errors-webpack-plugin": "^1.7.0", + "fs-extra": "^8.1.0", "fuzzy-search": "^3.0.2", "gh-pages": "^2.1.1", "html-loader": "^0.5.5", @@ -93,6 +98,7 @@ "js-yaml": "^3.13.1", "mini-css-extract-plugin": "^0.9.0", "npm-run-all": "^4.1.5", + "progress": "^2.0.3", "raw-loader": "^4.0.0", "style-loader": "^1.1.2", "tslint": "^5.20.1", diff --git a/src/components/contribute/Contribute.tsx b/src/components/contribute/Contribute.tsx index 64b2ef05..aec8e558 100644 --- a/src/components/contribute/Contribute.tsx +++ b/src/components/contribute/Contribute.tsx @@ -7,7 +7,7 @@ import * as R from 'ramda'; import yaml from 'js-yaml'; import {GRAY} from 'etc/colors'; -import {getStickerPackList, getStickerPack} from 'lib/stickers'; +import {getStickerPackDirectory, getStickerPack} from 'lib/stickers'; /** * Test URL: @@ -85,7 +85,7 @@ const validators = { const [, packId, packKey] = matches; - if (R.find(R.compose(R.propEq('id', packId), R.prop('meta')), await getStickerPackList())) { + if (R.find(R.pathEq(['meta', 'id'], packId) , await getStickerPackDirectory())) { return 'A sticker pack with that ID already exists in the directory.'; } diff --git a/src/components/home/Home.tsx b/src/components/home/Home.tsx index 1d0cd41a..d7f4231f 100644 --- a/src/components/home/Home.tsx +++ b/src/components/home/Home.tsx @@ -7,7 +7,7 @@ import StickerPackList from './SearchResults'; const HomeComponent: React.FunctionComponent = () => { return (
-

+

Welcome to Signal Stickers, the unofficial directory for Signal sticker packs. You can filter packs by title, author, or tags.

diff --git a/src/components/home/SearchResults.tsx b/src/components/home/SearchResults.tsx index c9a3d3ff..04fc36ac 100644 --- a/src/components/home/SearchResults.tsx +++ b/src/components/home/SearchResults.tsx @@ -26,7 +26,7 @@ const StickerPackList = styled.div` /** * How many items we will load each time loadMore() is called. */ -const PAGE_SIZE = 32; +const PAGE_SIZE = 64; const StickerPackListComponent = () => { @@ -73,7 +73,7 @@ const StickerPackListComponent = () => { ))} - + ); }; diff --git a/src/components/home/StickerPackPreviewCard.tsx b/src/components/home/StickerPackPreviewCard.tsx index efbd5b66..47248535 100644 --- a/src/components/home/StickerPackPreviewCard.tsx +++ b/src/components/home/StickerPackPreviewCard.tsx @@ -5,7 +5,7 @@ import Octicon from 'react-octicon'; import useAsyncEffect from 'use-async-effect'; import {StickerPack} from 'etc/types'; -import {getStickerInPack} from 'lib/stickers'; +import {getConvertedStickerInPack} from 'lib/stickers'; // ----- Props ----------------------------------------------------------------- @@ -29,7 +29,7 @@ const StickerPackPreviewCard = styled.div & {nsfw?: margin-bottom: 24px; margin-left: auto; margin-right: auto; - transition: transform 0.15s ease-in-out; + transition: transform 0.15s ease-in; filter: ${props => props.nsfw ? 'blur(4px)' : 'none'}; } @@ -84,7 +84,7 @@ const StickerPackPreviewCardComponent: React.FunctionComponent = props => useAsyncEffect(async () => { try { if (meta.id !== undefined) { // tslint:disable-line strict-type-predicates - const coverImage = await getStickerInPack(meta.id, meta.key, 'cover'); + const coverImage = await getConvertedStickerInPack(meta.id, meta.key, 'cover'); setCover(coverImage); } } catch (err) { diff --git a/src/components/layout/Footer.tsx b/src/components/layout/Footer.tsx index 85f059e0..41b57518 100644 --- a/src/components/layout/Footer.tsx +++ b/src/components/layout/Footer.tsx @@ -65,7 +65,7 @@ const FooterComponent: React.FunctionComponent = props => {

About

- Browse and download more than 100 sticker packs for Signal, the secure messenger. These + Browse and download more than 1000 sticker packs for Signal, the secure messenger. These stickers are created by the community; the maintainers of this website do not claim any rights. This site is not affiliated with Signal.

diff --git a/src/components/pack/Sticker.tsx b/src/components/pack/Sticker.tsx index 1ebc651b..8d269d08 100644 --- a/src/components/pack/Sticker.tsx +++ b/src/components/pack/Sticker.tsx @@ -4,7 +4,7 @@ import {styled} from 'linaria/react'; import Octicon from 'react-octicon'; import useAsyncEffect from 'use-async-effect'; -import {getStickerInPack, getEmojiForSticker} from 'lib/stickers'; +import {getConvertedStickerInPack, getEmojiForSticker} from 'lib/stickers'; // ----- Props ----------------------------------------------------------------- @@ -62,7 +62,7 @@ const StickerComponent: React.FunctionComponent = ({packId, packKey, stic stickerResult ] = await Promise.all([ getEmojiForSticker(packId, packKey, stickerId), - getStickerInPack(packId, packKey, stickerId) + getConvertedStickerInPack(packId, packKey, stickerId) ]); setEmoji(emojiResult); diff --git a/src/components/pack/StickerPackDetail.tsx b/src/components/pack/StickerPackDetail.tsx index fabf9068..ce7780fb 100644 --- a/src/components/pack/StickerPackDetail.tsx +++ b/src/components/pack/StickerPackDetail.tsx @@ -5,15 +5,12 @@ import {styled} from 'linaria/react'; import {darken} from 'polished'; // @ts-ignore (No type definitions exist for this package.) import Octicon from 'react-octicon'; -import * as R from 'ramda'; import useAsyncEffect from 'use-async-effect'; import {GRAY, SIGNAL_BLUE} from 'etc/colors'; -import {StickerPackManifest, StickerPackMetadata} from 'etc/types'; +import {StickerPack} from 'etc/types'; import useQuery from 'hooks/use-query'; -import ErrorWithCode from 'lib/error'; -import {getStickerPack, getStickerPackList} from 'lib/stickers'; -import {capitalizeFirst} from 'lib/utils'; +import {getStickerPack} from 'lib/stickers'; import Sticker from './Sticker'; import StickerPackError from './StickerPackError'; @@ -117,25 +114,11 @@ const StickerPackDetailComponent: React.FunctionComponent = () => { // Extract :packId from the URL. const {packId} = useParams(); - // Extract the optional "key" query param from the URL. - const key = useQuery().get('key'); - - - // This will be either the key extracted from a StickerPackMetadata object or, - // if the user is trying to load an unlisted sticker pack, the 'key' query - // param. - const [stickerPackKey, setStickerPackKey] = useState(); - - - // Sticker pack manifest from Signal. - const [stickerPack, setStickerPack] = useState(); - - - // Sticker pack metadata from stickerData.json. This will not be available if - // viewing an unlisted pack. - const [stickerPackMeta, setStickerPackMeta] = useState(); + const key = useQuery().get('key') || undefined; + // StickerPack object for the requested pack. + const [stickerPack, setStickerPack] = useState(); // One of many possible error codes we may catch when trying to load or // decrypt a sticker pack. This will be used to determine what error message @@ -144,42 +127,15 @@ const StickerPackDetailComponent: React.FunctionComponent = () => { /** - * [Effect] Set `stickerPack`, `stickerPackKey` and 'stickerPackMeta` when the - * component mounts. + * [Effect] Set `stickerPack` when the component mounts. */ useAsyncEffect(async () => { try { - // Because this value comes from a context, we may get rendered before - // it becomes available. if (!packId) { return; } - // Try to get stickerPack (including the pack's key) from stickerData.json. - const allPackMeta = R.map(R.prop('meta'), await getStickerPackList()); - const packMetadata = R.find(R.propEq('id', packId), allPackMeta); - - // If the sticker pack is listed in stickerData.json, set metadata and - // then fetch the pack's manifest from Signal using the key from - // metadata. - if (packMetadata) { - setStickerPackMeta(packMetadata); - setStickerPackKey(packMetadata.key); - setStickerPack(await getStickerPack(packId, packMetadata.key, true)); - return; - } - - // Then try to load and decrypt the manifest from Signal using the key - // from query params. - if (key) { - setStickerPackKey(key); - setStickerPack(await getStickerPack(packId, key)); - return; - } - - // If we couldn't find the pack in our directory and the user did not - // provide a `key` query param, throw. - throw new ErrorWithCode('NO_KEY_PROVIDED', 'No key provided.'); + setStickerPack(await getStickerPack(packId, key)); } catch (err) { if (err.code) { setStickerPackError(err.code); @@ -190,7 +146,7 @@ const StickerPackDetailComponent: React.FunctionComponent = () => { // ----- Render -------------------------------------------------------------- - if (!packId || !stickerPackKey || !stickerPack) { + if (!packId || !stickerPack) { // If an error code has been set, display an error alert to the user. if (stickerPackError) { switch (stickerPackError) { @@ -221,23 +177,26 @@ const StickerPackDetailComponent: React.FunctionComponent = () => { return null; // tslint:disable-line no-null-keyword } - const source = stickerPackMeta?.source || 'N/A'; - const numStickers = stickerPack.stickers.length; + const source = stickerPack.meta.source || 'N/A'; + const numStickers = stickerPack.manifest.stickers.length; // N.B. Signal allows strings containing only whitespace as authors. In these // cases, use 'Anonymous' instead. - const author = stickerPack.author.trim() ? stickerPack.author : 'Anonymous'; - const stickerPackTags = stickerPackMeta?.tags; - const addToSignalHref = `https://signal.art/addstickers/#pack_id=${packId}&pack_key=${stickerPackKey}`; + const author = stickerPack.manifest.author.trim() ? stickerPack.manifest.author : 'Anonymous'; + const stickerPackTags = stickerPack.meta.tags || []; + const addToSignalHref = `https://signal.art/addstickers/#pack_id=${packId}&pack_key=${stickerPack.meta.key}`; + + // TODO: Fix logic around displaying home button to better detect when we're + // viewing an unlisted sticker pack. return (
-
{stickerPack.title}
+
{stickerPack.manifest.title}
{author}
- {stickerPackMeta ? + {stickerPack.meta ? @@ -250,7 +209,7 @@ const StickerPackDetailComponent: React.FunctionComponent = () => {
- {stickerPackMeta ?
+ {stickerPack.meta ?
  • @@ -263,7 +222,7 @@ const StickerPackDetailComponent: React.FunctionComponent = () => { {numStickers}
    • - {stickerPackMeta.nsfw ?
  • + {stickerPack.meta.nsfw ?
  • NSFW
    • : null}
  • @@ -279,7 +238,7 @@ const StickerPackDetailComponent: React.FunctionComponent = () => {
    - {stickerPack.stickers.map(sticker => ())} + {stickerPack.manifest.stickers.map(sticker => ())}
    diff --git a/src/contexts/StickersContext.tsx b/src/contexts/StickersContext.tsx index e9fc94b2..3573ff0f 100644 --- a/src/contexts/StickersContext.tsx +++ b/src/contexts/StickersContext.tsx @@ -4,7 +4,7 @@ import useAsyncEffect from 'use-async-effect'; import {StickerPack} from 'etc/types'; import { - getStickerPackList, + getStickerPackDirectory, getStickerPack, fuzzySearchStickerPacks } from 'lib/stickers'; @@ -53,7 +53,7 @@ export const Provider = (props: PropsWithChildren<{}>) => { */ useAsyncEffect(async () => { // Load the set of sticker packs we need from stickerData.json. - const stickerPacks = await getStickerPackList(); + const stickerPacks = await getStickerPackDirectory(); // Set the canonical list of all sticker packs. setAllStickerPacks(stickerPacks); diff --git a/src/etc/Stickers.proto b/src/etc/Stickers.proto deleted file mode 100644 index b0e939bd..00000000 --- a/src/etc/Stickers.proto +++ /dev/null @@ -1,12 +0,0 @@ -message Pack { - message Sticker { - optional uint32 id = 1; - optional string emoji = 2; - } - - optional string title = 1; - optional string author = 2; - optional Sticker cover = 3; - repeated Sticker stickers = 4; -} - diff --git a/src/etc/stickers-proto.ts b/src/etc/stickers-proto.ts new file mode 100644 index 00000000..7ad7e8da --- /dev/null +++ b/src/etc/stickers-proto.ts @@ -0,0 +1,40 @@ +export default { + nested: { + Pack: { + fields: { + title: { + type: 'string', + id: 1 + }, + author: { + type: 'string', + id: 2 + }, + cover: { + type: 'Sticker', + id: 3 + }, + stickers: { + rule: 'repeated', + type: 'Sticker', + id: 4, + options: {} + } + }, + nested: { + Sticker: { + fields: { + id: { + type: 'uint32', + id: 1 + }, + emoji: { + type: 'string', + id: 2 + } + } + } + } + } + } +}; diff --git a/src/etc/types.ts b/src/etc/types.ts index 655dac51..75898313 100644 --- a/src/etc/types.ts +++ b/src/etc/types.ts @@ -1,26 +1,26 @@ -// ----- JSON Manifest --------------------------------------------------------- +// ----- YAML Manifest --------------------------------------------------------- /** - * Shape of the stickers.json manifest file. + * Shape of the stickers.yml manifest file. */ -export interface StickerPackJson { +export interface StickerPackYaml { [index: string]: { key: string; source: string; - tags: string; + tags: Array; nsfw?: boolean; }; } /** - * Shape of transformed objects when loaded from stickers.json such that the + * Shape of transformed objects when loaded from stickers.yml such that the * sticker pack ID is added to each object. */ export interface StickerPackMetadata { id: string; key: string; - source: string; - tags: Array; + source?: string; + tags?: Array; nsfw?: boolean; } @@ -64,9 +64,17 @@ export interface StickerPackManifest { // ----- Custom Objects -------------------------------------------------------- +/** + * A sticker pack contains all information about a sticker pack from + * stickers.yml (StickerPackMetadata) plus its manifest as fetched from the + * Signal API (StickerPackManifest). + * + * If the pack is "unlisted", its metadata + * will only contain the pack's id and key. + */ export interface StickerPack { /** - * All information about the sticker pack from stickers.json. + * All information about the sticker pack from stickers.yml. */ meta: StickerPackMetadata; @@ -75,3 +83,17 @@ export interface StickerPack { */ manifest: StickerPackManifest; } + + +/** + * A sticker pack partial is an object that contains all information for a + * sticker pack from stickers.yml plus its title and author, which are fetched + * from the Signal API. + * + * Sticker pack partials are used as the source of truth for searching, + * filtering, and displaying preview cards on the home page. + */ +export interface StickerPackPartial { + meta: StickerPack['meta']; + manifest: Pick; +} diff --git a/src/lib/convert.ts b/src/lib/convert-image.ts similarity index 95% rename from src/lib/convert.ts rename to src/lib/convert-image.ts index 26e97de1..46ac7949 100644 --- a/src/lib/convert.ts +++ b/src/lib/convert-image.ts @@ -67,10 +67,10 @@ export async function hasWebpSupport() { * webp-hero and returned as a base-64 encoded string. Both variants are * suitable for using in the "src" attribute of an img tag. */ -export async function convertImage(imageData: Uint8Array) { +export async function convertImage(rawImageData: Uint8Array) { if (await hasWebpSupportPromise) { // If the browser supports WebP, we don't need to convert it to PNG. - const base64Data = btoa(String.fromCharCode.apply(undefined, imageData)); + const base64Data = btoa(String.fromCharCode.apply(undefined, rawImageData)); return `data:image/webp;base64,${base64Data}`; } @@ -81,7 +81,8 @@ export async function convertImage(imageData: Uint8Array) { try { // @ts-ignore (`busy` is not an exposed member of WebpMachine.) await pWaitFor(() => webpConverter.busy === false); - return await webpConverter.decode(imageData); + + return await webpConverter.decode(rawImageData); } catch (err) { console.error(`[convertImage] Image conversion failed: ${err.message}`); throw err; diff --git a/src/lib/crypto.ts b/src/lib/crypto.ts index 85503aeb..a6c1e153 100644 --- a/src/lib/crypto.ts +++ b/src/lib/crypto.ts @@ -5,11 +5,21 @@ * pack data from Signal. Browser detection for browser support of the * SubtleCrypto API should probably be added at some point. * + * TODO: This module, along with signal.ts, should be refactored-out into a + * separate NPM package with separate browser/Node imports, which will allow us + * to make this code a bit cleaner. For now, this let's us use a consistent API + * for both environments. + * * See: https://developer.mozilla.org/en-US/docs/Web/API/SubtleCrypto */ +import crypto from 'crypto'; +import hkdf from 'js-crypto-hkdf'; import ErrorWithCode from 'lib/error'; +const IS_BROWSER = typeof window !== 'undefined'; // tslint:disable-line + + /** * [private] * @@ -34,52 +44,82 @@ function hexToArrayBuffer(hexString: string) { * See: https://developer.mozilla.org/en-US/docs/Web/API/SubtleCrypto/deriveKey */ async function deriveKeys(encodedKey: string) { - const masterKey = await window.crypto.subtle.importKey('raw', hexToArrayBuffer(encodedKey), 'HKDF', false, ['deriveKey']); - - const algorithm: HkdfParams = { - name: 'HKDF', - hash: 'SHA-256', - salt: new ArrayBuffer(32), - info: new TextEncoder().encode('Sticker Pack') - }; - - const derivedKeyAlgorithm = { - name: 'HMAC', - hash: 'SHA-256', - length: 512 - }; - - // @ts-ignore (The typedef for the SubtleCrypto API incorrectly states that - // we need an HkdfCtrParams object as our first param when in fact we need a - // HkdfParams object.) - const derivedKeys = await window.crypto.subtle.deriveKey(algorithm, masterKey, derivedKeyAlgorithm, true, ['verify']); - - const derivedKeyBytes = await window.crypto.subtle.exportKey('raw', derivedKeys); - return [derivedKeyBytes.slice(0, 32), derivedKeyBytes.slice(32, 64)]; + if (IS_BROWSER) { + const masterKey = await window.crypto.subtle.importKey('raw', hexToArrayBuffer(encodedKey), 'HKDF', false, ['deriveKey']); + + const algorithm: HkdfParams = { + name: 'HKDF', + hash: 'SHA-256', + salt: new ArrayBuffer(32), + info: new TextEncoder().encode('Sticker Pack') + }; + + const derivedKeyAlgorithm = { + name: 'HMAC', + hash: 'SHA-256', + length: 512 + }; + + // @ts-ignore (The typedef for the SubtleCrypto API incorrectly states that + // we need an HkdfCtrParams object as our first param when in fact we need a + // HkdfParams object.) + const derivedKeys = await window.crypto.subtle.deriveKey(algorithm, masterKey, derivedKeyAlgorithm, true, ['verify']); + + const derivedKeyBytes = await window.crypto.subtle.exportKey('raw', derivedKeys); + return [derivedKeyBytes.slice(0, 32), derivedKeyBytes.slice(32, 64)]; + } else { // tslint:disable-line unnecessary-else + const masterKey = Buffer.from(encodedKey, 'hex'); + const hash = 'SHA-256'; + const length = 512; + const info = 'Sticker Pack'; + const salt = new ArrayBuffer(32); + // @ts-ignore + const derivedKey = (await hkdf.compute(masterKey, hash, length, info, salt)).key; + return [derivedKey.slice(0, 32), derivedKey.slice(32, 64)]; + } } /** * Decrypts a manifest returned from the Signal API using a sticker pack's - * pack key, provided from stickers.json. + * pack key, provided from stickers.yml. */ export async function decryptManifest(encodedKey: string, rawManifest: any) { try { - const keys = await deriveKeys(encodedKey); - const encryptedManifest = new Uint8Array(rawManifest); - const theirIv = encryptedManifest.slice(0, 16); - const cipherTextBody = encryptedManifest.slice(16, encryptedManifest.byteLength - 32); - const theirMac = encryptedManifest.slice(encryptedManifest.byteLength - 32, encryptedManifest.byteLength); - const combinedCipherText = encryptedManifest.slice(0, encryptedManifest.byteLength - 32); - const macKey = await window.crypto.subtle.importKey('raw', keys[1], {name: 'HMAC', hash: {name: 'SHA-256'}}, false, ['verify', 'sign']); - const isValid = await window.crypto.subtle.verify('HMAC', macKey, theirMac, combinedCipherText); - - if (!isValid) { - throw new Error('MAC verification failed.'); - } + if (IS_BROWSER) { + const keys = await deriveKeys(encodedKey); + const encryptedManifest = new Uint8Array(rawManifest); + const theirIv = encryptedManifest.slice(0, 16); + const cipherTextBody = encryptedManifest.slice(16, encryptedManifest.byteLength - 32); + const theirMac = encryptedManifest.slice(encryptedManifest.byteLength - 32, encryptedManifest.byteLength); + const combinedCipherText = encryptedManifest.slice(0, encryptedManifest.byteLength - 32); + const macKey = await window.crypto.subtle.importKey('raw', keys[1], {name: 'HMAC', hash: {name: 'SHA-256'}}, false, ['verify', 'sign']); + const isValid = await window.crypto.subtle.verify('HMAC', macKey, theirMac, combinedCipherText); + + if (!isValid) { + throw new Error('MAC verification failed.'); + } + + const cipherKey = await window.crypto.subtle.importKey('raw', keys[0], 'AES-CBC', false, ['decrypt']); + return window.crypto.subtle.decrypt({name: 'AES-CBC', iv: theirIv}, cipherKey, cipherTextBody); + } else { // tslint:disable-line unnecessary-else + const [aesKey, hmacKey] = await deriveKeys(encodedKey); - const cipherKey = await window.crypto.subtle.importKey('raw', keys[0], 'AES-CBC', false, ['decrypt']); - return window.crypto.subtle.decrypt({name: 'AES-CBC', iv: theirIv}, cipherKey, cipherTextBody); + // rawManifest: IV || Ciphertext || truncated MAC(IV||Ciphertext) + const theirIv = rawManifest.slice(0, 16); + const cipherTextBody = rawManifest.slice(16, rawManifest.length - 32); + const theirMac = rawManifest.slice(rawManifest.byteLength - 32, rawManifest.byteLength).toString('hex'); + const combinedCipherText = rawManifest.slice(0, rawManifest.byteLength - 32); + + // Validate signature + const computedMac = crypto.createHmac('sha256', hmacKey).update(combinedCipherText).digest('hex'); + if (theirMac !== computedMac) { + throw new Error(`MAC verification failed.`); + } + + const decipher = crypto.createDecipheriv('aes-256-cbc', aesKey, theirIv); + return Buffer.concat([decipher.update(cipherTextBody), decipher.final()]); + } } catch (err) { throw ErrorWithCode.from('MANIFEST_DECRYPT', err); } diff --git a/src/lib/signal.ts b/src/lib/signal.ts new file mode 100644 index 00000000..5ad4dd8b --- /dev/null +++ b/src/lib/signal.ts @@ -0,0 +1,134 @@ +// ===== Signal Stickers Module ================================================ + +/** + * This module contains several functions for loading, fetching, decrypting, and + * caching sticker manifests and images from the Signal API. + */ +import axios from 'axios'; +import protobuf from 'protobufjs'; + +import StickersProto from 'etc/stickers-proto'; +import {StickerPackManifest} from 'etc/types'; +import {decryptManifest} from 'lib/crypto'; +import ErrorWithCode from 'lib/error'; + + +// ----- Locals ---------------------------------------------------------------- + +/** + * gRPC type definition for Signal's sticker pack manifests. + */ +const packMessage = protobuf.Root.fromJSON(StickersProto).root.lookupType('Pack'); + + +/** + * In-memory cache of sticker pack manifests. Helps us avoid making un-necessary + * network requests. + */ +const stickerPackManifestCache = new Map>(); + + +/** + * In-memory cache of sticker image data. Helps us avoid making un-necessary + * network requests. + */ +const stickerImageCache = new Map>(); + + +// ----- Functions ------------------------------------------------------------- + +/** + * Provided a key and an encrypted manifest from the Signal API, resolves with a + * decrypted and parsed manifest. + */ +async function parseManifest(key: string, rawManifest: any): Promise { + try { + const manifest = await decryptManifest(key, rawManifest); + const manifestData = new Uint8Array(manifest, 0, manifest.byteLength); + return packMessage.decode(manifestData) as unknown as StickerPackManifest; + } catch (err) { + throw new ErrorWithCode(err.code || 'MANIFEST_PARSE', `[parseManifest] ${err.stack}`); + } +} + + +/** + * Provided a sticker pack ID and key, queries the Signal API and resolves with + * a sticker pack manifest. + */ +export async function getStickerPackManifest(id: string, key: string): Promise { + if (!stickerPackManifestCache.has(id)) { + stickerPackManifestCache.set(id, new Promise(async (resolve, reject) => { + try { + const res = await axios({ + method: 'GET', + responseType: 'arraybuffer', + url: `https://cdn-ca.signal.org/stickers/${id}/manifest.proto` + }); + + const manifest = await parseManifest(key, res.data); + + resolve(manifest); + } catch (err) { + reject(err); + } + })); + } + + return stickerPackManifestCache.get(id) as Promise; +} + + +/** + * Provided a sticker pack ID and a sticker ID (or 'cover' for the pack's cover + * sticker) queries the Signal API and resolves with the raw WebP image data for + * the indicated sticker. + * + * Note: Web users who want to use this data to render an image will need to + * prefix this string with "data:image/webp;base64,". + */ +export async function getStickerInPack(id: string, key: string, stickerId: number | 'cover'): Promise { + const cacheKey = `${id}-${stickerId}`; + + if (!stickerImageCache.has(cacheKey)) { + stickerImageCache.set(cacheKey, new Promise(async (resolve, reject) => { + try { + const packManifest = await getStickerPackManifest(id, key); + const finalStickerId = stickerId === 'cover' ? packManifest.cover.id : stickerId; + + const res = await axios({ + method: 'GET', + responseType: 'arraybuffer', + url: `https://cdn-ca.signal.org/stickers/${id}/full/${finalStickerId}` + }); + + const stickerManifest = await decryptManifest(key, res.data); + const rawWebpData = new Uint8Array(stickerManifest, 0, stickerManifest.byteLength); + // const base64Data = btoa(String.fromCharCode.apply(undefined, arrayBufferView)); + resolve(rawWebpData); + } catch (err) { + reject(err); + } + })); + } + + return stickerImageCache.get(cacheKey) as Promise; +} + + +/** + * Provided a sticker pack ID, key, and sticker ID, returns the emoji associated + * with the sticker. + */ +export async function getEmojiForSticker(id: string, key: string, stickerId: number | 'cover'): Promise { + const packManifest = await getStickerPackManifest(id, key); + const finalStickerId = stickerId === 'cover' ? packManifest.cover.id : stickerId; + + const sticker = packManifest.stickers.find(curSticker => curSticker.id === finalStickerId); + + if (!sticker) { + throw new Error(`Sticker pack ${id} has no sticker with ID ${stickerId}.`); + } + + return sticker.emoji; +} diff --git a/src/lib/stickers.ts b/src/lib/stickers.ts index 9c5a9274..65cc7343 100644 --- a/src/lib/stickers.ts +++ b/src/lib/stickers.ts @@ -12,39 +12,38 @@ import axios from 'axios'; import FuzzySearch from 'fuzzy-search'; import LocalForage from 'localforage'; -import protobuf from 'protobufjs'; import * as R from 'ramda'; -import { - StickerPackJson, - StickerPackMetadata, - StickerPackManifest, - StickerPack, - Sticker -} from 'etc/types'; -import StickersProto from 'etc/Stickers.proto'; -import {convertImage} from 'lib/convert'; -import {decryptManifest} from 'lib/crypto'; +import {StickerPack, StickerPackPartial} from 'etc/types'; +import {convertImage} from 'lib/convert-image'; import ErrorWithCode from 'lib/error'; +import { + getStickerPackManifest, + getStickerInPack, + getEmojiForSticker +} from 'lib/signal'; + // ----- Locals ---------------------------------------------------------------- /** - * Module-local gRPC client used to parse sticker pack manifests from the Signal - * CDN. + * Promise that will resolve with the list of sticker packs enumerated in + * stickers.yaml. This collection will contain only those data from a + * StickerPack that we want to search on or that we need to display a sticker + * pack preview card. We use a promise here rather than the array itself to + * ensure that if multiple calls to getStickerPackDirectory are made before the + * initial request for stickerData.json resolves, we only make a single request + * and only populate the directory once. */ -const protobufClient = protobuf.parse(StickersProto).root; +let stickerPackDirectoryPromise: Promise>; + /** - * Module-local in-memory copy of stickerData.json, ensures we only load it once. + * In-memory cache of StickerPack objects. */ -let stickerPackListCache: Array = []; +const stickerPackCache = new Map(); - /** - * Module-local in-memory cache used for sticker pack data from the Signal API. - */ -let stickerPackCache = new Map(); /** * Module-local browser-storage-backed cache used for sticker image data. @@ -57,78 +56,63 @@ const stickerImageCache = LocalForage.createInstance({ // ----- Functions ------------------------------------------------------------- -async function warmCachesIfNecessary() { - if (stickerPackListCache.length !== 0) { - return; - } - - const res = await axios({ - method: 'GET', - url: 'stickerData.json' - }); - - stickerPackListCache = res.data as Array; - - // Warm sticker manifest map. - stickerPackCache = R.reduce((result, value) => { - result.set(value.meta.id, value.manifest); - return result; - }, new Map(), stickerPackListCache); -} - /** - * Loads and transforms stickers.json, which is the source of truth regarding - * all sticker packs included in the application. + * Resolves with a list of StickerPackPartial objects. */ -export async function getStickerPackList(): Promise> { - await warmCachesIfNecessary(); - return stickerPackListCache; -} - +export async function getStickerPackDirectory(): Promise> { + if (!stickerPackDirectoryPromise) { + stickerPackDirectoryPromise = new Promise(async (resolve, reject) => { + const res = await axios.request>({ + method: 'GET', + url: 'stickerData.json' + }); -/** - * Provided a key and an encrypted manifest from the Signal API, resolves with a - * decrypted and parsed manifest. - */ -export async function parseManifest(key: string, rawManifest: any): Promise { - try { - const manifest = await decryptManifest(key, rawManifest); - const PackMessage = protobufClient.lookupType('Pack'); - const manifestData = new Uint8Array(manifest, 0, manifest.byteLength); - return PackMessage.decode(manifestData) as unknown as StickerPackManifest; - } catch (err) { - throw new ErrorWithCode(err.code || 'MANIFEST_PARSE', `[parseManifest] ${err.message}`); + resolve(res.data); + }); } + + return stickerPackDirectoryPromise; } /** - * Provided a sticker pack ID and key, queries the Signal API and resolves with - * a parsed manifest. + * Provided a sticker pack ID and optional key, queries the Signal API and + * resolves with 'full' StickerPack object. */ -export async function getStickerPack(id: string, key: string, fetchStickersIfNecessary: boolean | false): Promise { +export async function getStickerPack(id: string, key?: string): Promise { try { - await warmCachesIfNecessary(); + if (!stickerPackCache.has(id)) { + const directory = await getStickerPackDirectory(); - const cacheKey = id; + // Build the metadata object using information from a StickerPackPartial + // in the directory or, if the requested sticker pack is unlisted, just + // the id and key. + const partial = R.find(R.pathEq(['meta', 'id'], id), directory); + const meta = partial ? partial.meta : {id, key}; - const hasCachedPack = stickerPackCache.has(cacheKey); - const hasStickers = hasCachedPack && stickerPackCache.get(cacheKey).stickers; + const finalKey = key ?? meta.key; - if (!hasCachedPack || (fetchStickersIfNecessary && !hasStickers)) { - const res = await axios({ - method: 'GET', - responseType: 'arraybuffer', - url: `https://cdn-ca.signal.org/stickers/${id}/manifest.proto` - }); + if (!finalKey) { + throw new ErrorWithCode('NO_KEY_PROVIDED', `No key provided for unlisted pack: ${id}.`); + } - const manifest = await parseManifest(key, res.data); - stickerPackCache.set(cacheKey, manifest); + const manifest = await getStickerPackManifest(id, finalKey); + + const stickerPack: StickerPack = { + meta, + manifest + }; + + stickerPackCache.set(id, stickerPack); } - return stickerPackCache.get(cacheKey) as StickerPackManifest; + return stickerPackCache.get(id) as StickerPack; } catch (err) { - throw new ErrorWithCode(err.code, `[getStickerPack] ${err.message}`); + if (err.isAxiosError && err.response.status === 403) { + throw new ErrorWithCode('MANIFEST_DECRYPT', `[getStickerPack] ${err.stack}`); + } + + throw new ErrorWithCode(err.code, `[getStickerPack] ${err.stack}`); } } @@ -138,13 +122,13 @@ export async function getStickerPack(id: string, key: string, fetchStickersIfNec * sticker) queries the Signal API and resolves with a base-64 encoded string * representing the image data for the indicated sticker. */ -export async function getStickerInPack(id: string, key: string, stickerId: number | 'cover'): Promise { +export async function getConvertedStickerInPack(id: string, key: string, stickerId: number | 'cover'): Promise { try { const cacheKey = `${id}-${stickerId}`; - const item = await stickerImageCache.getItem(cacheKey); + const imageFromCache = await stickerImageCache.getItem(cacheKey); - if (!item) { + if (!imageFromCache) { // Before we can make the request, we need to get the pack's information // using getStickerPack. const stickerPack = await getStickerPack(id, key); @@ -153,44 +137,18 @@ export async function getStickerInPack(id: string, key: string, stickerId: numbe throw new Error(`[getStickerInPack] Unable to get sticker ${stickerId} in pack ${id}.`); } - const finalStickerId = stickerId === 'cover' ? stickerPack.cover.id : stickerId; + const finalStickerId = stickerId === 'cover' ? stickerPack.manifest.cover.id : stickerId; - const res = await axios({ - method: 'GET', - responseType: 'arraybuffer', - url: `https://cdn-ca.signal.org/stickers/${id}/full/${finalStickerId}` - }); - - const manifest = await decryptManifest(key, res.data); - const arrayBufferView = new Uint8Array(manifest, 0, manifest.byteLength); - const convertedImage = await convertImage(arrayBufferView); + const rawImageData = await getStickerInPack(id, key, finalStickerId); + const convertedImage = await convertImage(rawImageData); await stickerImageCache.setItem(cacheKey, convertedImage); - } - return item || await stickerImageCache.getItem(cacheKey); - } catch (err) { - throw new Error(`[getStickerInPack] Error getting sticker: ${err.message}`); - } -} - - -/** - * Provided a sticker pack ID, key, and sticker ID, returns the emoji associated - * with the sticker. - */ -export async function getEmojiForSticker(id: string, key: string, stickerId: number | 'cover') { - try { - const stickerPack = await getStickerPack(id, key); - const finalStickerId = stickerId === 'cover' ? stickerPack.cover.id : stickerId; - const sticker = R.find(R.propEq('id', finalStickerId), stickerPack.stickers); - - if (!sticker) { - throw new Error(`Sticker pack ${id} has no sticker with ID ${stickerId}.`); + return convertedImage; } - return sticker.emoji; + return await stickerImageCache.getItem(cacheKey); } catch (err) { - throw new Error(`[getEmojiForSticker] ${err.stack}`); + throw new Error(`[getStickerInPack] Error getting sticker: ${err.message}`); } } @@ -204,3 +162,8 @@ export function fuzzySearchStickerPacks(needle: string, haystack: Array> { + const stickerPackPartials: Array = []; + const stickerPackYaml: StickerPackYaml = yaml.safeLoad(await fs.readFile(inputFile, {encoding: 'utf8'})); + const stickerPackEntries = Object.entries(stickerPackYaml); - const decipher = crypto.createDecipheriv('aes-256-cbc', aesKey, theirIv); - return Buffer.concat([decipher.update(cipherTextBody), decipher.final()]); -} + const gitRoot = await findUp('.git', {type: 'directory', cwd: __dirname}); -/** - * Provided a key and an encrypted manifest from the Signal API, resolves with a - * decrypted and parsed manifest. - */ -async function parseManifest(key: string, rawManifest: any): Promise { - try { - const manifest = await decryptManifest(key, rawManifest); - const PackMessage = protobufClient.lookupType('Pack'); - const manifestData = new Uint8Array(manifest, 0, manifest.byteLength); - return PackMessage.decode(manifestData) as unknown as StickerPackManifest; - } catch (err) { - throw new ErrorWithCode(err.code || 'MANIFEST_PARSE', `[parseManifest] ${err.message}`); + if (!gitRoot) { + throw new Error('Not in a git repository.'); } -} -/** - * Waits for a random number of ms between min and max before resolving the promise. - */ -async function randomDelay(minMs: number, maxMs: number) { - const delayMs = Math.round(Math.random() * (maxMs - minMs)) + minMs; - return new Promise(resolve => setTimeout(resolve, delayMs)); // tslint:disable-line no-string-based-set-timeout -} + const cacheDir = path.resolve(gitRoot, '..', '.sticker-pack-cache'); -/** - * Provided a sticker pack ID and key, queries the Signal API and resolves with - * a parsed manifest. - */ -async function getStickerPack(id: string, key: string, retriesRemaining = 5): Promise { - try { - const res = await axios({ - method: 'GET', - responseType: 'arraybuffer', - url: `https://cdn-ca.signal.org/stickers/${id}/manifest.proto` - }); + // Create the cache directory if it does not exist. + await fs.ensureDir(cacheDir); - return await parseManifest(key, res.data); - } catch (err) { - // If the error was due to tiemout and we have retries remaining, retry - if (err.code === 'ECONNABORTED' && retriesRemaining > 0) { - // pause before retrying. - await randomDelay(250, 500); - console.log(`Retrying fetch for ${id}, attempts remaining: ${retriesRemaining - 1}`); - return getStickerPack(id, key, retriesRemaining - 1); - } - - throw new ErrorWithCode(err.code, `[getStickerPack] ${err.message}`); - } + console.log('[FetchStickerDataPlugin] Downloading sticker pack manifests.'); + + const bar = new ProgressBar('[FetchStickerDataPlugin] [:bar] :current / :total', { + total: stickerPackEntries.length, + width: 80, + clear: true, + head: '>' + }); + + await requestQueue.addAll(stickerPackEntries.map(([id, meta], index) => { + return async () => { + const cachePath = path.resolve(cacheDir, `${id}.json`); + const cacheHasPack = await fs.pathExists(cachePath); + let stickerPackPartial: StickerPackPartial; + + if (cacheHasPack) { + stickerPackPartial = await fs.readJson(cachePath); + } else { + const stickerPackManifest = await getStickerPackManifest(id, meta.key); + + stickerPackPartial = { + meta: {...meta, id}, + // To keep the size of the generated JSON file small, only extract + // the properties from the manifest that we need to generate a list + // of search results. + manifest: R.pick(['title', 'author'], stickerPackManifest) + }; + + await fs.writeJson(cachePath, stickerPackPartial); + } + + stickerPackPartials.push(stickerPackPartial); + bar.tick(); + }; + })); + + console.log('[FetchStickerDataPlugin] Done.\n'); + + return stickerPackPartials; } + /** - * Query the manifest for all known sticker packs. + * Options accepted by FetchStickerDataPlugin. */ -async function getAllStickerPacks(): Promise { - return Promise.all(R.map(async ([id, value]) => { - try { - // Signal's service will ignore our request if we fetch - // too many packs too rapidly so we stagger our timing. - await randomDelay(0, 1000); - const manifest = await getStickerPack(id, value.key); - return { - meta: {id, ...value}, - manifest: { - title: manifest.title, - author: manifest.author, - cover: manifest.cover - } - }; - } catch (err) { - throw new ErrorWithCode(err.code, `[getAllStickerPacks] ${err.message}`); - } - }, Object.entries(stickers as StickerPackJson))); +export interface FetchStickerDataPluginOptions { + inputFile: string; + outputFile: string; } + export default class FetchStickerDataPlugin { - constructor({filename}) { - this.filename = filename; + /** + * Name of the input file, which should be a YAML document consisting of an + * array of StickerPackMetadata objects. + */ + inputFile: string; + + /** + * Name of the output file, which will be a JSON document containing an array + * of StickerPackPartial objects. + */ + outputFile: string; + + + constructor({inputFile, outputFile}: FetchStickerDataPluginOptions) { + this.inputFile = inputFile; + this.outputFile = outputFile; } - apply(compiler) { - compiler.plugin('emit', async (compilation, done) => { - const json = JSON.stringify(await getAllStickerPacks()); - compilation.assets[this.filename] = { + + apply(compiler: webpack.Compiler) { + compiler.hooks.emit.tapPromise('FetchStickerDataPlugin', async compilation => { + const json = JSON.stringify(await getAllStickerPacks(this.inputFile), null, 2); + + compilation.assets[this.outputFile] = { source: () => json, size: () => json.length }; - done(); }); } }