From d6f8fa62935de9b444974ce937ee88172bcd69bc Mon Sep 17 00:00:00 2001 From: Tim Oxley Date: Tue, 16 Mar 2021 15:53:27 -0400 Subject: [PATCH 01/28] test: Fix SubscriberResends uncontrolled setTimeout. Fix enableAutoConnect type. --- src/Connection.ts | 4 ++-- test/integration/SubscriberResends.test.ts | 4 +--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/Connection.ts b/src/Connection.ts index 92c3fdb52..84bf7040a 100644 --- a/src/Connection.ts +++ b/src/Connection.ts @@ -425,7 +425,7 @@ export default class Connection extends EventEmitter { } } - enableAutoDisconnect(autoDisconnect = true) { + enableAutoDisconnect(autoDisconnect: boolean | number = true) { let delay if (typeof autoDisconnect === 'number') { delay = autoDisconnect @@ -443,7 +443,7 @@ export default class Connection extends EventEmitter { } } - enableAutoConnect(autoConnect = true) { + enableAutoConnect(autoConnect: boolean | number = true) { autoConnect = !!autoConnect // eslint-disable-line no-param-reassign if (this.options.autoConnect && !autoConnect) { this.didDisableAutoConnect = true diff --git a/test/integration/SubscriberResends.test.ts b/test/integration/SubscriberResends.test.ts index 611f47382..1d45b5ed9 100644 --- a/test/integration/SubscriberResends.test.ts +++ b/test/integration/SubscriberResends.test.ts @@ -286,8 +286,7 @@ describeRepeats('resends', () => { client.connection.enableAutoDisconnect(false) }) client.connection.enableAutoConnect() - // @ts-expect-error - client.connection.enableAutoDisconnect(600) // set 0 delay + client.connection.enableAutoDisconnect(0) // set 0 delay const sub = await subscriber.resend({ streamId: stream.id, last: published.length, @@ -475,7 +474,6 @@ describeRepeats('resends', () => { it('can end inside resend', async () => { const unsubscribeEvents: any[] = [] - // @ts-expect-error client.connection.on(ControlMessage.TYPES.UnsubscribeResponse, (m) => { unsubscribeEvents.push(m) }) From 4ea7856f9ccc417f1dcf5811b30efe6bd54d7abb Mon Sep 17 00:00:00 2001 From: Tim Oxley Date: Wed, 17 Mar 2021 09:20:27 -0400 Subject: [PATCH 02/28] test(subscribe, iterators): Fix error handling. --- src/subscribe/index.ts | 4 ++-- src/utils/iterators.js | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/subscribe/index.ts b/src/subscribe/index.ts index 800fea9fb..2e7fa5c5b 100644 --- a/src/subscribe/index.ts +++ b/src/subscribe/index.ts @@ -49,7 +49,7 @@ export class Subscription extends Emitter { /** @internal */ debug - constructor(client: StreamrClient, opts: Todo, onFinally = defaultOnFinally) { + constructor(client: StreamrClient, opts: Todo, onFinally: MaybeAsync<(err?: any) => void> = defaultOnFinally) { super() this.client = client this.options = validateOptions(opts) @@ -601,7 +601,7 @@ export class Subscriber { return this.subscriptions.count(options) } - async subscribe(opts: StreamPartDefinition, onFinally?: MaybeAsync<(err?: any) => void>) { + async subscribe(opts: StreamPartDefinition, onFinally?: Todo) { return this.subscriptions.add(opts, onFinally) } diff --git a/src/utils/iterators.js b/src/utils/iterators.js index 603a6216a..179349caa 100644 --- a/src/utils/iterators.js +++ b/src/utils/iterators.js @@ -374,6 +374,7 @@ export function pipeline(iterables = [], onFinally = defaultOnFinally, { end, .. prev.id = prev.id || 'inter-' + nextIterable.id nextIterable.from(prev, { end }) } + yield* nextIterable }()), async (err) => { if (!error && err && error !== err) { From 4769eba6cf036ebc1c150771b2a81e8041f84aa6 Mon Sep 17 00:00:00 2001 From: Tim Oxley Date: Tue, 23 Feb 2021 16:59:24 -0500 Subject: [PATCH 03/28] Add initial version of GroupKey persistence. WIP. --- package-lock.json | 200 ++++++++++++++++++++++++++++++---- package.json | 1 + src/stream/BrowserStore.ts | 78 +++++++++++++ src/stream/KeyExchange.js | 14 ++- src/stream/PersistentStore.ts | 68 ++++++++++++ 5 files changed, 336 insertions(+), 25 deletions(-) create mode 100644 src/stream/BrowserStore.ts create mode 100644 src/stream/PersistentStore.ts diff --git a/package-lock.json b/package-lock.json index 606086c35..d3889b407 100644 --- a/package-lock.json +++ b/package-lock.json @@ -3750,6 +3750,32 @@ "uri-js": "^4.2.2" } }, + "ajv-formats": { + "version": "1.6.1", + "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-1.6.1.tgz", + "integrity": "sha512-4CjkH20If1lhR5CGtqkrVg3bbOtFEG80X9v6jDOIUhbzzbB+UzPBGy8GQhUNVZ0yvMHdMpawCOcy5ydGMsagGQ==", + "requires": { + "ajv": "^7.0.0" + }, + "dependencies": { + "ajv": { + "version": "7.2.4", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-7.2.4.tgz", + "integrity": "sha512-nBeQgg/ZZA3u3SYxyaDvpvDtgZ/EZPF547ARgZBrG9Bhu1vKDwAIjtIf+sDtJUKa2zOcEbmRLBRSyMraS/Oy1A==", + "requires": { + "fast-deep-equal": "^3.1.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2", + "uri-js": "^4.2.2" + } + }, + "json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==" + } + } + }, "ajv-keywords": { "version": "3.5.2", "resolved": "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-3.5.2.tgz", @@ -3929,6 +3955,13 @@ "integrity": "sha512-yG89F0j9B4B0MKIcFyWWxnpZPLaNTjCj4tkE3fjbAoo0qmpGw0PYYqSbX/4ebnd9Icn8ZgK4K1fvDyEtW1JYtQ==", "requires": { "pvutils": "^1.0.17" + }, + "dependencies": { + "pvutils": { + "version": "1.0.17", + "resolved": "https://registry.npmjs.org/pvutils/-/pvutils-1.0.17.tgz", + "integrity": "sha512-wLHYUQxWaXVQvKnwIDWFVKDJku9XDCvyhhxoq8dc5MFdIlRenyPI9eSfEtcvgHgD7FlvCyGAlWgOzRnZD99GZQ==" + } } }, "assert-plus": { @@ -3995,6 +4028,11 @@ "integrity": "sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg==", "dev": true }, + "atomically": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/atomically/-/atomically-1.7.0.tgz", + "integrity": "sha512-Xcz9l0z7y9yQ9rdDaxlmaI4uJHf/T8g9hOEzJcsEqX2SjCj4J20uK7+ldkDHMbpJDK76wF7xEIgxc/vSlsfw5w==" + }, "available-typed-arrays": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/available-typed-arrays/-/available-typed-arrays-1.0.2.tgz", @@ -5268,6 +5306,73 @@ "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=", "dev": true }, + "conf": { + "version": "9.0.2", + "resolved": "https://registry.npmjs.org/conf/-/conf-9.0.2.tgz", + "integrity": "sha512-rLSiilO85qHgaTBIIHQpsv8z+NnVfZq3cKuYNCXN1AOqPzced0GWZEe/A517VldRLyQYXUMyV+vszavE2jSAqw==", + "requires": { + "ajv": "^7.0.3", + "ajv-formats": "^1.5.1", + "atomically": "^1.7.0", + "debounce-fn": "^4.0.0", + "dot-prop": "^6.0.1", + "env-paths": "^2.2.0", + "json-schema-typed": "^7.0.3", + "make-dir": "^3.1.0", + "onetime": "^5.1.2", + "pkg-up": "^3.1.0", + "semver": "^7.3.4" + }, + "dependencies": { + "ajv": { + "version": "7.2.4", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-7.2.4.tgz", + "integrity": "sha512-nBeQgg/ZZA3u3SYxyaDvpvDtgZ/EZPF547ARgZBrG9Bhu1vKDwAIjtIf+sDtJUKa2zOcEbmRLBRSyMraS/Oy1A==", + "requires": { + "fast-deep-equal": "^3.1.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2", + "uri-js": "^4.2.2" + } + }, + "dot-prop": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-6.0.1.tgz", + "integrity": "sha512-tE7ztYzXHIeyvc7N+hR3oi7FIbf/NIjVP9hmAt3yMXzrQ072/fpjGLx2GxNxGxUl5V73MEqYzioOMoVhGMJ5cA==", + "requires": { + "is-obj": "^2.0.0" + } + }, + "json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==" + }, + "make-dir": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-3.1.0.tgz", + "integrity": "sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw==", + "requires": { + "semver": "^6.0.0" + }, + "dependencies": { + "semver": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==" + } + } + }, + "semver": { + "version": "7.3.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.5.tgz", + "integrity": "sha512-PoeGJYh8HK4BTO/a9Tf6ZG3veo/A7ZVsYrSA6J8ny9nb3B1VrpkuN+z9OE5wfE5p6H4LchYZsegiQgbJD94ZFQ==", + "requires": { + "lru-cache": "^6.0.0" + } + } + } + }, "confusing-browser-globals": { "version": "1.0.10", "resolved": "https://registry.npmjs.org/confusing-browser-globals/-/confusing-browser-globals-1.0.10.tgz", @@ -5567,6 +5672,14 @@ "whatwg-url": "^8.0.0" } }, + "debounce-fn": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/debounce-fn/-/debounce-fn-4.0.0.tgz", + "integrity": "sha512-8pYCQiL9Xdcg0UPSD3d+0KMlOjp+KGU5EPwYddgzQ7DATsg4fuUDjQtsYLmWjnk2obnNHgV3vE2Y4jejSOJVBQ==", + "requires": { + "mimic-fn": "^3.0.0" + } + }, "debug": { "version": "4.3.2", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.2.tgz", @@ -5983,6 +6096,11 @@ "ansi-colors": "^4.1.1" } }, + "env-paths": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/env-paths/-/env-paths-2.2.1.tgz", + "integrity": "sha512-+h1lkLKhZMTYjog1VEpJNG7NZJWcuc2DDk/qsqSTRRCOXiLjeQ1d1/udrUGhqMxUgAlwKNZ0cf2uqan5GLuS2A==" + }, "envinfo": { "version": "7.8.1", "resolved": "https://registry.npmjs.org/envinfo/-/envinfo-7.8.1.tgz", @@ -7017,8 +7135,7 @@ "fast-deep-equal": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", - "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", - "dev": true + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==" }, "fast-glob": { "version": "3.2.5", @@ -8240,8 +8357,7 @@ "is-obj": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/is-obj/-/is-obj-2.0.0.tgz", - "integrity": "sha512-drqDG3cbczxxEJRoOXcOjtdp1J/lyp1mNn0xaznRs8+muBhgQcrnbspox5X5fOw0HnMnbfDzvnEMEtqDEJEo8w==", - "dev": true + "integrity": "sha512-drqDG3cbczxxEJRoOXcOjtdp1J/lyp1mNn0xaznRs8+muBhgQcrnbspox5X5fOw0HnMnbfDzvnEMEtqDEJEo8w==" }, "is-path-cwd": { "version": "2.2.0", @@ -10141,6 +10257,11 @@ "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", "dev": true }, + "json-schema-typed": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/json-schema-typed/-/json-schema-typed-7.0.3.tgz", + "integrity": "sha512-7DE8mpG+/fVw+dTpjbxnx47TaMnDfOI1jwft9g1VybltZCduyRQPJPvc+zzKY9WPHxhPWczyFuYa6I8Mw4iU5A==" + }, "json-stable-stringify-without-jsonify": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz", @@ -10476,7 +10597,6 @@ "version": "6.0.0", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", - "dev": true, "requires": { "yallist": "^4.0.0" } @@ -11608,7 +11728,6 @@ "version": "5.1.2", "resolved": "https://registry.npmjs.org/onetime/-/onetime-5.1.2.tgz", "integrity": "sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==", - "dev": true, "requires": { "mimic-fn": "^2.1.0" }, @@ -11616,8 +11735,7 @@ "mimic-fn": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", - "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==", - "dev": true + "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==" } } }, @@ -11897,8 +12015,7 @@ "p-try": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", - "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", - "dev": true + "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==" }, "pac-proxy-agent": { "version": "4.1.0", @@ -12090,6 +12207,54 @@ "find-up": "^4.0.0" } }, + "pkg-up": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/pkg-up/-/pkg-up-3.1.0.tgz", + "integrity": "sha512-nDywThFk1i4BQK4twPQ6TA4RT8bDY96yeuCVBWL3ePARCiEKDRSrNGbFIgUJpLp+XeIR65v8ra7WuJOFUBtkMA==", + "requires": { + "find-up": "^3.0.0" + }, + "dependencies": { + "find-up": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-3.0.0.tgz", + "integrity": "sha512-1yD6RmLI1XBfxugvORwlck6f75tYL+iR0jqwsOrOxMZyGYqUuDhJ0l4AXdO1iX/FTs9cBAMEk1gWSEx1kSbylg==", + "requires": { + "locate-path": "^3.0.0" + } + }, + "locate-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-3.0.0.tgz", + "integrity": "sha512-7AO748wWnIhNqAuaty2ZWHkQHRSNfPVIsPIfwEOWO22AmaoVrWavlOcMR5nzTLNYvp36X220/maaRsrec1G65A==", + "requires": { + "p-locate": "^3.0.0", + "path-exists": "^3.0.0" + } + }, + "p-limit": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", + "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", + "requires": { + "p-try": "^2.0.0" + } + }, + "p-locate": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-3.0.0.tgz", + "integrity": "sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ==", + "requires": { + "p-limit": "^2.0.0" + } + }, + "path-exists": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz", + "integrity": "sha1-zg6+ql94yxiSXqfYENe1mwEP1RU=" + } + } + }, "platform": { "version": "1.3.6", "resolved": "https://registry.npmjs.org/platform/-/platform-1.3.6.tgz", @@ -12279,8 +12444,7 @@ "punycode": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", - "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==", - "dev": true + "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==" }, "pvtsutils": { "version": "1.1.2", @@ -12290,11 +12454,6 @@ "tslib": "^2.1.0" } }, - "pvutils": { - "version": "1.0.17", - "resolved": "https://registry.npmjs.org/pvutils/-/pvutils-1.0.17.tgz", - "integrity": "sha512-wLHYUQxWaXVQvKnwIDWFVKDJku9XDCvyhhxoq8dc5MFdIlRenyPI9eSfEtcvgHgD7FlvCyGAlWgOzRnZD99GZQ==" - }, "q": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/q/-/q-1.5.1.tgz", @@ -12787,8 +12946,7 @@ "require-from-string": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", - "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", - "dev": true + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==" }, "require-main-filename": { "version": "2.0.0", @@ -14547,7 +14705,6 @@ "version": "4.4.1", "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==", - "dev": true, "requires": { "punycode": "^2.1.0" } @@ -15219,8 +15376,7 @@ "yallist": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", - "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", - "dev": true + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" }, "yaml": { "version": "1.10.2", diff --git a/package.json b/package.json index 737ef0a8b..5daa95179 100644 --- a/package.json +++ b/package.json @@ -150,6 +150,7 @@ "@ethersproject/transactions": "^5.1.1", "@ethersproject/wallet": "^5.1.0", "@ethersproject/web": "^5.1.0", + "conf": "^9.0.2", "debug": "^4.3.2", "eventemitter3": "^4.0.7", "lodash": "^4.17.21", diff --git a/src/stream/BrowserStore.ts b/src/stream/BrowserStore.ts new file mode 100644 index 000000000..5068b4603 --- /dev/null +++ b/src/stream/BrowserStore.ts @@ -0,0 +1,78 @@ +import type { PersistentStorage } from './PersistentStore' + +export default class BrowserStorage implements PersistentStorage { + id: string + constructor(id: string) { + this.id = id + } + + private getData() { + return JSON.parse(window.localStorage.get(this.id) || {}) || {} + } + + private setData(value: any) { + return window.localStorage.set(this.id, JSON.stringify(value)) + } + + private mergeData(value: any) { + const data = this.getData() + return window.localStorage.set(this.id, JSON.stringify({ + ...data, + ...value + })) + } + + has(key: string) { + return !!this.getData()[key] + } + + get(key: string) { + return this.getData()[key] + } + + keys() { + return Object.keys(this.getData())[Symbol.iterator]() + } + + values() { + return Object.values(this.getData())[Symbol.iterator]() + } + + entries() { + return Object.entries(this.getData())[Symbol.iterator]() + } + + forEach(...args: Parameters['forEach']>) { + return new Map(Object.entries(this.getData())).forEach(...args) + } + + set(key: string, value: any) { + return this.mergeData({ + [key]: value, + }) + } + + delete(key: string) { + const data = this.getData() + delete data[key] + return this.setData(data) + } + + clear() { + return this.setData({}) + } + + get size() { + const data = this.getData() + return Object.keys(data).length + } + + [Symbol.iterator]() { + return new Map(Object.entries(this.getData()))[Symbol.iterator]() + } + + get [Symbol.toStringTag]() { + return this.constructor.name + } +} + diff --git a/src/stream/KeyExchange.js b/src/stream/KeyExchange.js index 778f78d30..bfad4c5fc 100644 --- a/src/stream/KeyExchange.js +++ b/src/stream/KeyExchange.js @@ -6,6 +6,7 @@ import Scaffold from '../utils/Scaffold' import { validateOptions } from './utils' import EncryptionUtil, { GroupKey } from './Encryption' +import PersistentStore from './PersistentStore' const { StreamMessage, GroupKeyRequest, GroupKeyResponse, GroupKeyErrorResponse, EncryptedGroupKey @@ -58,8 +59,12 @@ export function getKeyExchangeStreamId(address) { return `${KEY_EXCHANGE_STREAM_PREFIX}/${address.toLowerCase()}` } -function GroupKeyStore({ groupKeys = new Map() }) { - const store = new Map() +function GroupKeyStore({ id, groupKeys = new Map() }) { + if (!id) { + throw new Error('GroupKeyStore missing id') + } + + const store = new PersistentStore(id) groupKeys.forEach((value, key) => { store.set(key, value) }) @@ -298,7 +303,9 @@ async function PublisherKeyExhangeSubscription(client, getGroupKeyStore) { export function PublisherKeyExhange(client, { groupKeys = {} } = {}) { let enabled = true + const clientId = client.getPublisherId() const getGroupKeyStore = mem((streamId) => GroupKeyStore({ + id: `${clientId}:${streamId}`, groupKeys: parseGroupKeys(groupKeys[streamId]) }), { cacheKey([maybeStreamId]) { @@ -404,8 +411,9 @@ async function SubscriberKeyExhangeSubscription(client, getGroupKeyStore, encryp export function SubscriberKeyExchange(client, { groupKeys = {} } = {}) { let enabled = true const encryptionUtil = new EncryptionUtil(client.options.keyExchange) - + const clientId = client.getPublisherId() const getGroupKeyStore = mem((streamId) => GroupKeyStore({ + id: clientId, groupKeys: parseGroupKeys(groupKeys[streamId]) }), { cacheKey([maybeStreamId]) { diff --git a/src/stream/PersistentStore.ts b/src/stream/PersistentStore.ts new file mode 100644 index 000000000..3d47539fd --- /dev/null +++ b/src/stream/PersistentStore.ts @@ -0,0 +1,68 @@ +import Conf from 'conf' + +export interface PersistentStorage extends Map { + id: string +} + +export default class ServerStorage implements PersistentStorage { + id: string + config: Conf + constructor(id: string) { + this.id = id + this.config = new Conf({ + projectName: 'streamr-client', + configName: id, + }) + } + + has(key: string) { + return this.config.has(key) + } + + get(key: string) { + return this.config.get(key) + } + + keys() { + return Object.keys(this.config.store)[Symbol.iterator]() + } + + values() { + return Object.values(this.config.store)[Symbol.iterator]() + } + + entries() { + return Object.entries(this.config.store)[Symbol.iterator]() + } + + forEach(...args: Parameters['forEach']>) { + return new Map(Object.entries(this.config.store)).forEach(...args) + } + + set(key: string, value: any) { + this.config.set(key, value) + return this + } + + delete(key: string) { + const had = this.config.has(key) + this.config.delete(key) + return had + } + + clear() { + return this.config.clear() + } + + get size() { + return this.config.size + } + + [Symbol.iterator]() { + return new Map(Object.entries(this.config.store))[Symbol.iterator]() + } + + get [Symbol.toStringTag]() { + return this.constructor.name + } +} From a069df5336b8b12d3d341f3548a9e25f16d67d5d Mon Sep 17 00:00:00 2001 From: Tim Oxley Date: Tue, 23 Feb 2021 17:21:44 -0500 Subject: [PATCH 04/28] Fix browser shimming of PersistentStore. --- src/stream/BrowserStore.ts | 8 ++++---- webpack.config.js | 1 + 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/stream/BrowserStore.ts b/src/stream/BrowserStore.ts index 5068b4603..25d76b787 100644 --- a/src/stream/BrowserStore.ts +++ b/src/stream/BrowserStore.ts @@ -3,20 +3,20 @@ import type { PersistentStorage } from './PersistentStore' export default class BrowserStorage implements PersistentStorage { id: string constructor(id: string) { - this.id = id + this.id = `BrowserStorage:${id}` } private getData() { - return JSON.parse(window.localStorage.get(this.id) || {}) || {} + return JSON.parse(window.localStorage.getItem(this.id) || '{}') || {} } private setData(value: any) { - return window.localStorage.set(this.id, JSON.stringify(value)) + return window.localStorage.setItem(this.id, JSON.stringify(value)) } private mergeData(value: any) { const data = this.getData() - return window.localStorage.set(this.id, JSON.stringify({ + return window.localStorage.setItem(this.id, JSON.stringify({ ...data, ...value })) diff --git a/webpack.config.js b/webpack.config.js index 2f46fa51a..ef36e33aa 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -103,6 +103,7 @@ module.exports = (env, argv) => { 'node-fetch': path.resolve(__dirname, './src/shim/node-fetch.js'), 'node-webcrypto-ossl': path.resolve(__dirname, 'src/shim/crypto.js'), 'streamr-client-protocol': path.resolve(__dirname, 'node_modules/streamr-client-protocol/dist/src'), + [path.resolve(__dirname, 'src/stream/PersistentStore')]: path.resolve(__dirname, 'src/stream/BrowserStore'), } }, plugins: [ From fa49c41e93d820df68e8a2ecde51a023e47af890 Mon Sep 17 00:00:00 2001 From: Tim Oxley Date: Tue, 23 Feb 2021 20:13:20 -0500 Subject: [PATCH 05/28] Fix BrowserStore method return values. --- src/stream/BrowserStore.ts | 10 ++++++++-- webpack.config.js | 1 + 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/stream/BrowserStore.ts b/src/stream/BrowserStore.ts index 25d76b787..61cac65f7 100644 --- a/src/stream/BrowserStore.ts +++ b/src/stream/BrowserStore.ts @@ -47,15 +47,21 @@ export default class BrowserStorage implements PersistentStorage { } set(key: string, value: any) { - return this.mergeData({ + this.mergeData({ [key]: value, }) + return this } delete(key: string) { + if (!this.has(key)) { + return false + } + const data = this.getData() delete data[key] - return this.setData(data) + this.setData(data) + return true } clear() { diff --git a/webpack.config.js b/webpack.config.js index ef36e33aa..500fb67c7 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -103,6 +103,7 @@ module.exports = (env, argv) => { 'node-fetch': path.resolve(__dirname, './src/shim/node-fetch.js'), 'node-webcrypto-ossl': path.resolve(__dirname, 'src/shim/crypto.js'), 'streamr-client-protocol': path.resolve(__dirname, 'node_modules/streamr-client-protocol/dist/src'), + // swap out PersistentStore for browser [path.resolve(__dirname, 'src/stream/PersistentStore')]: path.resolve(__dirname, 'src/stream/BrowserStore'), } }, From 1eeeac1f120e7c460ffb9af288af185c7739b134 Mon Sep 17 00:00:00 2001 From: Tim Oxley Date: Tue, 23 Feb 2021 20:28:09 -0500 Subject: [PATCH 06/28] Prevent premature access to client.publisherId in unauthenticated clients. --- src/stream/KeyExchange.js | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/stream/KeyExchange.js b/src/stream/KeyExchange.js index bfad4c5fc..93cbc9853 100644 --- a/src/stream/KeyExchange.js +++ b/src/stream/KeyExchange.js @@ -303,9 +303,8 @@ async function PublisherKeyExhangeSubscription(client, getGroupKeyStore) { export function PublisherKeyExhange(client, { groupKeys = {} } = {}) { let enabled = true - const clientId = client.getPublisherId() const getGroupKeyStore = mem((streamId) => GroupKeyStore({ - id: `${clientId}:${streamId}`, + id: `${client.getPublisherId()}:${streamId}`, groupKeys: parseGroupKeys(groupKeys[streamId]) }), { cacheKey([maybeStreamId]) { @@ -411,9 +410,8 @@ async function SubscriberKeyExhangeSubscription(client, getGroupKeyStore, encryp export function SubscriberKeyExchange(client, { groupKeys = {} } = {}) { let enabled = true const encryptionUtil = new EncryptionUtil(client.options.keyExchange) - const clientId = client.getPublisherId() const getGroupKeyStore = mem((streamId) => GroupKeyStore({ - id: clientId, + id: `${client.getPublisherId()}:${streamId}`, groupKeys: parseGroupKeys(groupKeys[streamId]) }), { cacheKey([maybeStreamId]) { From ff256add5e0a971f7f6bebeda4a4a5c1a4231e49 Mon Sep 17 00:00:00 2001 From: Tim Oxley Date: Tue, 23 Feb 2021 16:59:24 -0500 Subject: [PATCH 07/28] Add initial version of GroupKey persistence. WIP. --- src/stream/BrowserStore.ts | 18 ++++++------------ src/stream/KeyExchange.js | 6 ++++-- 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/src/stream/BrowserStore.ts b/src/stream/BrowserStore.ts index 61cac65f7..5068b4603 100644 --- a/src/stream/BrowserStore.ts +++ b/src/stream/BrowserStore.ts @@ -3,20 +3,20 @@ import type { PersistentStorage } from './PersistentStore' export default class BrowserStorage implements PersistentStorage { id: string constructor(id: string) { - this.id = `BrowserStorage:${id}` + this.id = id } private getData() { - return JSON.parse(window.localStorage.getItem(this.id) || '{}') || {} + return JSON.parse(window.localStorage.get(this.id) || {}) || {} } private setData(value: any) { - return window.localStorage.setItem(this.id, JSON.stringify(value)) + return window.localStorage.set(this.id, JSON.stringify(value)) } private mergeData(value: any) { const data = this.getData() - return window.localStorage.setItem(this.id, JSON.stringify({ + return window.localStorage.set(this.id, JSON.stringify({ ...data, ...value })) @@ -47,21 +47,15 @@ export default class BrowserStorage implements PersistentStorage { } set(key: string, value: any) { - this.mergeData({ + return this.mergeData({ [key]: value, }) - return this } delete(key: string) { - if (!this.has(key)) { - return false - } - const data = this.getData() delete data[key] - this.setData(data) - return true + return this.setData(data) } clear() { diff --git a/src/stream/KeyExchange.js b/src/stream/KeyExchange.js index 93cbc9853..bfad4c5fc 100644 --- a/src/stream/KeyExchange.js +++ b/src/stream/KeyExchange.js @@ -303,8 +303,9 @@ async function PublisherKeyExhangeSubscription(client, getGroupKeyStore) { export function PublisherKeyExhange(client, { groupKeys = {} } = {}) { let enabled = true + const clientId = client.getPublisherId() const getGroupKeyStore = mem((streamId) => GroupKeyStore({ - id: `${client.getPublisherId()}:${streamId}`, + id: `${clientId}:${streamId}`, groupKeys: parseGroupKeys(groupKeys[streamId]) }), { cacheKey([maybeStreamId]) { @@ -410,8 +411,9 @@ async function SubscriberKeyExhangeSubscription(client, getGroupKeyStore, encryp export function SubscriberKeyExchange(client, { groupKeys = {} } = {}) { let enabled = true const encryptionUtil = new EncryptionUtil(client.options.keyExchange) + const clientId = client.getPublisherId() const getGroupKeyStore = mem((streamId) => GroupKeyStore({ - id: `${client.getPublisherId()}:${streamId}`, + id: clientId, groupKeys: parseGroupKeys(groupKeys[streamId]) }), { cacheKey([maybeStreamId]) { From 020ddb59ca9509889627f77b776090f70dd0239c Mon Sep 17 00:00:00 2001 From: Tim Oxley Date: Thu, 18 Mar 2021 14:39:50 -0400 Subject: [PATCH 08/28] fix(keyexchange): Correct localStorage method calls. --- src/stream/BrowserStore.ts | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/src/stream/BrowserStore.ts b/src/stream/BrowserStore.ts index 5068b4603..e2db609cc 100644 --- a/src/stream/BrowserStore.ts +++ b/src/stream/BrowserStore.ts @@ -3,20 +3,20 @@ import type { PersistentStorage } from './PersistentStore' export default class BrowserStorage implements PersistentStorage { id: string constructor(id: string) { - this.id = id + this.id = `BrowserStorage:${id}` } private getData() { - return JSON.parse(window.localStorage.get(this.id) || {}) || {} + return JSON.parse(window.localStorage.getItem(this.id) || '{}') || {} } private setData(value: any) { - return window.localStorage.set(this.id, JSON.stringify(value)) + return window.localStorage.setItem(this.id, JSON.stringify(value)) } private mergeData(value: any) { const data = this.getData() - return window.localStorage.set(this.id, JSON.stringify({ + return window.localStorage.setItem(this.id, JSON.stringify({ ...data, ...value })) @@ -47,15 +47,21 @@ export default class BrowserStorage implements PersistentStorage { } set(key: string, value: any) { - return this.mergeData({ + this.mergeData({ [key]: value, }) + return this } delete(key: string) { + if (!this.has(key)) { + return false + } + const data = this.getData() delete data[key] - return this.setData(data) + this.setData(data) + return true } clear() { @@ -75,4 +81,3 @@ export default class BrowserStorage implements PersistentStorage { return this.constructor.name } } - From 4c550f64fb11ad606a75275d25865b1a74d0676e Mon Sep 17 00:00:00 2001 From: Tim Oxley Date: Fri, 19 Mar 2021 13:53:14 -0400 Subject: [PATCH 09/28] test: Fix connection.on event value. Fix test typing. --- test/integration/SubscriberResends.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/SubscriberResends.test.ts b/test/integration/SubscriberResends.test.ts index 1d45b5ed9..2097f9b87 100644 --- a/test/integration/SubscriberResends.test.ts +++ b/test/integration/SubscriberResends.test.ts @@ -474,7 +474,7 @@ describeRepeats('resends', () => { it('can end inside resend', async () => { const unsubscribeEvents: any[] = [] - client.connection.on(ControlMessage.TYPES.UnsubscribeResponse, (m) => { + client.connection.on(String(ControlMessage.TYPES.UnsubscribeResponse), (m) => { unsubscribeEvents.push(m) }) const sub = await subscriber.resendSubscribe({ From b5c708e0dd132d726c88d8e3fc1d9b10fac23b82 Mon Sep 17 00:00:00 2001 From: Tim Oxley Date: Mon, 22 Mar 2021 14:30:10 -0400 Subject: [PATCH 10/28] fix(keyexchange): Use sync client.id instead of async publisherId in cache key. --- src/stream/KeyExchange.js | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/stream/KeyExchange.js b/src/stream/KeyExchange.js index bfad4c5fc..d4644bb71 100644 --- a/src/stream/KeyExchange.js +++ b/src/stream/KeyExchange.js @@ -303,9 +303,8 @@ async function PublisherKeyExhangeSubscription(client, getGroupKeyStore) { export function PublisherKeyExhange(client, { groupKeys = {} } = {}) { let enabled = true - const clientId = client.getPublisherId() const getGroupKeyStore = mem((streamId) => GroupKeyStore({ - id: `${clientId}:${streamId}`, + id: `${client.id}:${streamId}`, groupKeys: parseGroupKeys(groupKeys[streamId]) }), { cacheKey([maybeStreamId]) { @@ -411,9 +410,8 @@ async function SubscriberKeyExhangeSubscription(client, getGroupKeyStore, encryp export function SubscriberKeyExchange(client, { groupKeys = {} } = {}) { let enabled = true const encryptionUtil = new EncryptionUtil(client.options.keyExchange) - const clientId = client.getPublisherId() const getGroupKeyStore = mem((streamId) => GroupKeyStore({ - id: clientId, + id: client.id, groupKeys: parseGroupKeys(groupKeys[streamId]) }), { cacheKey([maybeStreamId]) { From f1afa5bad58b9a9f3a5a2cbad4ae78c5cc934a12 Mon Sep 17 00:00:00 2001 From: Tim Oxley Date: Mon, 3 May 2021 12:48:14 -0400 Subject: [PATCH 11/28] types: Add types to KeyExchange. --- src/publish/Encrypt.ts | 1 - src/stream/Encryption.ts | 31 ++- src/stream/{KeyExchange.js => KeyExchange.ts} | 197 ++++++++++-------- 3 files changed, 131 insertions(+), 98 deletions(-) rename src/stream/{KeyExchange.js => KeyExchange.ts} (75%) diff --git a/src/publish/Encrypt.ts b/src/publish/Encrypt.ts index ebee57bae..26ff27482 100644 --- a/src/publish/Encrypt.ts +++ b/src/publish/Encrypt.ts @@ -39,7 +39,6 @@ export default function Encrypt(client: StreamrClient) { if (streamMessage.messageType !== StreamMessage.MESSAGE_TYPES.MESSAGE) { return } - const [groupKey, nextGroupKey] = await getPublisherKeyExchange().useGroupKey(stream.id) await EncryptionUtil.encryptStreamMessage(streamMessage, groupKey, nextGroupKey) } diff --git a/src/stream/Encryption.ts b/src/stream/Encryption.ts index 161c484ec..ea7bdbae3 100644 --- a/src/stream/Encryption.ts +++ b/src/stream/Encryption.ts @@ -11,10 +11,10 @@ import { uuid } from '../utils' const { StreamMessage, EncryptedGroupKey } = MessageLayer -export class UnableToDecryptError extends Error { +export class StreamMessageProcessingError extends Error { streamMessage: MessageLayer.StreamMessage constructor(message = '', streamMessage: MessageLayer.StreamMessage) { - super(`Unable to decrypt. ${message} ${util.inspect(streamMessage)}`) + super(`Could not process. ${message} ${util.inspect(streamMessage)}`) this.streamMessage = streamMessage if (Error.captureStackTrace) { Error.captureStackTrace(this, this.constructor) @@ -22,9 +22,18 @@ export class UnableToDecryptError extends Error { } } +export class UnableToDecryptError extends StreamMessageProcessingError { + constructor(message = '', streamMessage: MessageLayer.StreamMessage) { + super(`Unable to decrypt. ${message} ${util.inspect(streamMessage)}`, streamMessage) + if (Error.captureStackTrace) { + Error.captureStackTrace(this, this.constructor) + } + } +} + class InvalidGroupKeyError extends Error { groupKey: GroupKey | any - constructor(message: string, groupKey?: GroupKey) { + constructor(message: string, groupKey?: any) { super(message) this.groupKey = groupKey if (Error.captureStackTrace) { @@ -59,6 +68,8 @@ function GroupKeyObjectFromProps(data: GroupKeyProps | GroupKeyObject) { interface GroupKey extends GroupKeyObject {} +export type GroupKeyish = GroupKey | GroupKeyObject | ConstructorParameters + // eslint-disable-next-line no-redeclare class GroupKey { static InvalidGroupKeyError = InvalidGroupKeyError @@ -148,7 +159,7 @@ class GroupKey { return new GroupKey(id, keyBytes) } - static from(maybeGroupKey: GroupKey | GroupKeyObject | ConstructorParameters) { + static from(maybeGroupKey: GroupKeyish) { if (!maybeGroupKey || typeof maybeGroupKey !== 'object') { throw new InvalidGroupKeyError(`Group key must be object ${util.inspect(maybeGroupKey)}`) } @@ -224,10 +235,15 @@ class EncryptionUtilBase { } } - /* + /** * Returns a Buffer or a hex String */ - static encryptWithPublicKey(plaintextBuffer: Uint8Array, publicKey: crypto.KeyLike, outputInHex = false) { + /* eslint-disable no-dupe-class-members */ + static encryptWithPublicKey(plaintextBuffer: Uint8Array, publicKey: crypto.KeyLike, outputInHex: true): string + // These overrides tell ts outputInHex returns string + static encryptWithPublicKey(plaintextBuffer: Uint8Array, publicKey: crypto.KeyLike): string + static encryptWithPublicKey(plaintextBuffer: Uint8Array, publicKey: crypto.KeyLike, outputInHex: false): Buffer + static encryptWithPublicKey(plaintextBuffer: Uint8Array, publicKey: crypto.KeyLike, outputInHex: boolean = false) { this.validatePublicKey(publicKey) const ciphertextBuffer = crypto.publicEncrypt(publicKey, plaintextBuffer) if (outputInHex) { @@ -235,11 +251,12 @@ class EncryptionUtilBase { } return ciphertextBuffer } + /* eslint-disable no-dupe-class-members */ /* * Both 'data' and 'groupKey' must be Buffers. Returns a hex string without the '0x' prefix. */ - static encrypt(data: Uint8Array, groupKey: GroupKey) { + static encrypt(data: Uint8Array, groupKey: GroupKey): string { GroupKey.validate(groupKey) const iv = crypto.randomBytes(16) // always need a fresh IV when using CTR mode const cipher = crypto.createCipheriv('aes-256-ctr', groupKey.data, iv) diff --git a/src/stream/KeyExchange.js b/src/stream/KeyExchange.ts similarity index 75% rename from src/stream/KeyExchange.js rename to src/stream/KeyExchange.ts index d4644bb71..32cd8b39c 100644 --- a/src/stream/KeyExchange.js +++ b/src/stream/KeyExchange.ts @@ -1,16 +1,15 @@ -import { MessageLayer, Errors } from 'streamr-client-protocol' +import { + StreamMessage, GroupKeyRequest, GroupKeyResponse, GroupKeyErrorResponse, EncryptedGroupKey, Errors +} from 'streamr-client-protocol' import mem from 'mem' import { uuid, Defer } from '../utils' import Scaffold from '../utils/Scaffold' import { validateOptions } from './utils' -import EncryptionUtil, { GroupKey } from './Encryption' -import PersistentStore from './PersistentStore' - -const { - StreamMessage, GroupKeyRequest, GroupKeyResponse, GroupKeyErrorResponse, EncryptedGroupKey -} = MessageLayer +import EncryptionUtil, { GroupKey, GroupKeyish, StreamMessageProcessingError } from './Encryption' +import type { Subscription } from '../subscribe' +import { StreamrClient } from '../StreamrClient' const KEY_EXCHANGE_STREAM_PREFIX = 'SYSTEM/keyexchange' @@ -21,7 +20,8 @@ export function isKeyExchangeStream(id = '') { } class InvalidGroupKeyRequestError extends ValidationError { - constructor(...args) { + code: string + constructor(...args: ConstructorParameters) { super(...args) this.code = 'INVALID_GROUP_KEY_REQUEST' if (Error.captureStackTrace) { @@ -52,36 +52,37 @@ class InvalidContentTypeError extends Error { } */ -export function getKeyExchangeStreamId(address) { +type Address = string +type GroupKeyId = string + +function getKeyExchangeStreamId(address: Address) { if (isKeyExchangeStream(address)) { return address // prevent ever double-handling } return `${KEY_EXCHANGE_STREAM_PREFIX}/${address.toLowerCase()}` } -function GroupKeyStore({ id, groupKeys = new Map() }) { - if (!id) { - throw new Error('GroupKeyStore missing id') - } +type GroupKeyStoreOptions = { + id: string, + groupKeys: [GroupKeyId, GroupKey][] +} - const store = new PersistentStore(id) - groupKeys.forEach((value, key) => { - store.set(key, value) - }) +function GroupKeyStore({ groupKeys }: GroupKeyStoreOptions) { + const store = new Map(groupKeys) - let currentGroupKeyId // current key id if any - const nextGroupKeys = [] // the keys to use next, disappears if not actually used. Max queue size 2 + let currentGroupKeyId: GroupKeyId | undefined // current key id if any + const nextGroupKeys: GroupKey[] = [] // the keys to use next, disappears if not actually used. Max queue size 2 store.forEach((groupKey) => { - GroupKey.validate(GroupKey.from(groupKey)) + GroupKey.validate(groupKey) // use last init key as current currentGroupKeyId = groupKey.id }) - function storeKey(groupKey) { + function storeKey(groupKey: GroupKey) { GroupKey.validate(groupKey) - if (store.has(groupKey.id)) { - const existingKey = GroupKey.from(store.get(groupKey.id)) + const existingKey = store.get(groupKey.id) + if (existingKey) { if (!existingKey.equals(groupKey)) { throw new GroupKey.InvalidGroupKeyError( `Trying to add groupKey ${groupKey.id} but key exists & is not equivalent to new GroupKey: ${groupKey}.`, @@ -99,17 +100,17 @@ function GroupKeyStore({ id, groupKeys = new Map() }) { } return { - has(groupKeyId) { - if (currentGroupKeyId === groupKeyId) { return true } + has(id: GroupKeyId) { + if (currentGroupKeyId === id) { return true } - if (nextGroupKeys.some((nextKey) => nextKey.id === groupKeyId)) { return true } + if (nextGroupKeys.some((nextKey) => nextKey.id === id)) { return true } - return store.has(groupKeyId) + return store.has(id) }, isEmpty() { return nextGroupKeys.length === 0 && store.size === 0 }, - useGroupKey() { + useGroupKey(): Promise<[GroupKey, GroupKey | undefined]> { const nextGroupKey = nextGroupKeys.pop() switch (true) { // First use of group key on this stream, no current key. Make next key current. @@ -141,15 +142,13 @@ function GroupKeyStore({ id, groupKeys = new Map() }) { } // Generate & use new key if none already set. default: { - this.rotateGroupKey() - return this.useGroupKey() - } + this.rotateGroupKey() + return this.useGroupKey() } + } }, - get(groupKeyId) { - const groupKey = store.get(groupKeyId) - if (!groupKey) { return undefined } - return GroupKey.from(groupKey) + get(id: GroupKeyId) { + return store.get(id) }, clear() { currentGroupKeyId = undefined @@ -159,10 +158,10 @@ function GroupKeyStore({ id, groupKeys = new Map() }) { rotateGroupKey() { return this.setNextGroupKey(GroupKey.generate()) }, - add(groupKey) { + add(groupKey: GroupKey) { return storeKey(groupKey) }, - setNextGroupKey(newKey) { + setNextGroupKey(newKey: GroupKey) { GroupKey.validate(newKey) nextGroupKeys.unshift(newKey) nextGroupKeys.length = Math.min(nextGroupKeys.length, 2) @@ -176,16 +175,21 @@ function GroupKeyStore({ id, groupKeys = new Map() }) { } } -function parseGroupKeys(groupKeys = {}) { - return new Map(Object.entries(groupKeys || {}).map(([key, value]) => { +type GroupKeyStorage = ReturnType +type GroupKeysSerialized = Record + +function parseGroupKeys(groupKeys: GroupKeysSerialized = {}): Map { + return new Map(Object.entries(groupKeys || {}).map(([key, value]) => { if (!value || !key) { return null } return [key, GroupKey.from(value)] - }).filter(Boolean)) + }).filter(Boolean) as []) } -function waitForSubMessage(sub, matchFn) { +type MessageMatch = (content: any, streamMessage: StreamMessage) => boolean + +function waitForSubMessage(sub: Subscription, matchFn: MessageMatch) { const task = Defer() - const onMessage = (content, streamMessage) => { + const onMessage = (content: any, streamMessage: StreamMessage) => { try { if (matchFn(content, streamMessage)) { task.resolve(streamMessage) @@ -199,13 +203,13 @@ function waitForSubMessage(sub, matchFn) { task.finally(() => { sub.off('message', onMessage) sub.off('error', task.reject) - }).catch(() => {}) // prevent unhandled rejection + }).catch(() => {}) // important: prevent unchained finally cleanup causing unhandled rejection return task } -async function subscribeToKeyExchangeStream(client, onKeyExchangeMessage) { +async function subscribeToKeyExchangeStream(client: StreamrClient, onKeyExchangeMessage: (msg: any, streamMessage: StreamMessage) => void) { const { options } = client - if ((!options.auth.privateKey && !options.auth.ethereum) || !options.keyExchange) { + if ((!options.auth!.privateKey && !options.auth!.ethereum) || !options.keyExchange) { return Promise.resolve() } @@ -218,7 +222,7 @@ async function subscribeToKeyExchangeStream(client, onKeyExchangeMessage) { return sub } -async function catchKeyExchangeError(client, streamMessage, fn) { +async function catchKeyExchangeError(client: StreamrClient, streamMessage: StreamMessage, fn: (...args: any[]) => Promise) { try { return await fn() } catch (error) { @@ -235,8 +239,8 @@ async function catchKeyExchangeError(client, streamMessage, fn) { } } -async function PublisherKeyExhangeSubscription(client, getGroupKeyStore) { - async function onKeyExchangeMessage(_parsedContent, streamMessage) { +async function PublisherKeyExhangeSubscription(client: StreamrClient, getGroupKeyStore: (streamId: string) => GroupKeyStorage) { + async function onKeyExchangeMessage(_parsedContent: any, streamMessage: StreamMessage) { return catchKeyExchangeError(client, streamMessage, async () => { if (streamMessage.messageType !== StreamMessage.MESSAGE_TYPES.GROUP_KEY_REQUEST) { return Promise.resolve() @@ -254,9 +258,9 @@ async function PublisherKeyExhangeSubscription(client, getGroupKeyStore) { if (!groupKey) { return null // will be filtered out } - - return new EncryptedGroupKey(id, EncryptionUtil.encryptWithPublicKey(groupKey.data, rsaPublicKey, true)) - }).filter(Boolean) + const key = EncryptionUtil.encryptWithPublicKey(groupKey.data, rsaPublicKey, true) + return new EncryptedGroupKey(id, key) + }).filter(Boolean) as EncryptedGroupKey[] client.debug('Publisher: Subscriber requested groupKeys: %d. Got: %d. %o', groupKeyIds.length, encryptedGroupKeys.length, { subscriberId, @@ -268,7 +272,6 @@ async function PublisherKeyExhangeSubscription(client, getGroupKeyStore) { streamId, requestId, encryptedGroupKeys, - encryptionType: StreamMessage.ENCRYPTION_TYPES.RSA, }) // hack overriding toStreamMessage method to set correct encryption type @@ -284,15 +287,15 @@ async function PublisherKeyExhangeSubscription(client, getGroupKeyStore) { } const sub = await subscribeToKeyExchangeStream(client, onKeyExchangeMessage) - sub.on('error', (err) => { - if (!err.streamMessage) { + sub.on('error', (err: Error | StreamMessageProcessingError) => { + if (!('streamMessage' in err)) { return // do nothing } // wrap error and translate into ErrorResponse. catchKeyExchangeError(client, err.streamMessage, () => { // eslint-disable-line promise/no-promise-in-callback // rethrow so catchKeyExchangeError handles it - throw new InvalidGroupKeyRequestError(err.message, err.streamMessage) + throw new InvalidGroupKeyRequestError(err.message) }).catch((unexpectedError) => { sub.emit('error', unexpectedError) }) @@ -301,18 +304,22 @@ async function PublisherKeyExhangeSubscription(client, getGroupKeyStore) { return sub } -export function PublisherKeyExhange(client, { groupKeys = {} } = {}) { +type KeyExhangeOptions = { + groupKeys?: Record +} + +export function PublisherKeyExhange(client: StreamrClient, { groupKeys = {} }: KeyExhangeOptions = {}) { let enabled = true const getGroupKeyStore = mem((streamId) => GroupKeyStore({ id: `${client.id}:${streamId}`, - groupKeys: parseGroupKeys(groupKeys[streamId]) + groupKeys: [...parseGroupKeys(groupKeys![streamId]).entries()], }), { cacheKey([maybeStreamId]) { const { streamId } = validateOptions(maybeStreamId) return streamId } }) - let sub + let sub: Subscription | undefined const next = Scaffold([ async () => { sub = await PublisherKeyExhangeSubscription(client, getGroupKeyStore) @@ -323,22 +330,22 @@ export function PublisherKeyExhange(client, { groupKeys = {} } = {}) { await cancelTask } } - ], () => enabled) + ], async () => enabled) - function rotateGroupKey(streamId) { + function rotateGroupKey(streamId: string) { if (!enabled) { return } const groupKeyStore = getGroupKeyStore(streamId) groupKeyStore.rotateGroupKey() } - function setNextGroupKey(streamId, groupKey) { + function setNextGroupKey(streamId: string, groupKey: GroupKey) { if (!enabled) { return } const groupKeyStore = getGroupKeyStore(streamId) groupKeyStore.setNextGroupKey(groupKey) } - async function useGroupKey(streamId) { + async function useGroupKey(streamId: string) { await next() if (!enabled) { return undefined } const groupKeyStore = getGroupKeyStore(streamId) @@ -346,7 +353,7 @@ export function PublisherKeyExhange(client, { groupKeys = {} } = {}) { return groupKeyStore.useGroupKey() } - function hasAnyGroupKey(streamId) { + function hasAnyGroupKey(streamId: string) { const groupKeyStore = getGroupKeyStore(streamId) return !groupKeyStore.isEmpty() } @@ -375,16 +382,20 @@ export function PublisherKeyExhange(client, { groupKeys = {} } = {}) { } } -async function getGroupKeysFromStreamMessage(streamMessage, encryptionUtil) { +async function getGroupKeysFromStreamMessage(streamMessage: StreamMessage, encryptionUtil: EncryptionUtil) { const { encryptedGroupKeys } = GroupKeyResponse.fromArray(streamMessage.getParsedContent()) return Promise.all(encryptedGroupKeys.map(async (encryptedGroupKey) => ( new GroupKey(encryptedGroupKey.groupKeyId, await encryptionUtil.decryptWithPrivateKey(encryptedGroupKey.encryptedGroupKeyHex, true)) ))) } -async function SubscriberKeyExhangeSubscription(client, getGroupKeyStore, encryptionUtil) { - let sub - async function onKeyExchangeMessage(_parsedContent, streamMessage) { +async function SubscriberKeyExhangeSubscription( + client: StreamrClient, + getGroupKeyStore: (streamId: string) => GroupKeyStorage, + encryptionUtil: EncryptionUtil +) { + let sub: Subscription + async function onKeyExchangeMessage(_parsedContent: any, streamMessage: StreamMessage) { try { const { messageType } = streamMessage const { MESSAGE_TYPES } = StreamMessage @@ -407,12 +418,13 @@ async function SubscriberKeyExhangeSubscription(client, getGroupKeyStore, encryp return sub } -export function SubscriberKeyExchange(client, { groupKeys = {} } = {}) { +export function SubscriberKeyExchange(client: StreamrClient, { groupKeys = {} }: KeyExhangeOptions = {}) { let enabled = true const encryptionUtil = new EncryptionUtil(client.options.keyExchange) + const getGroupKeyStore = mem((streamId) => GroupKeyStore({ id: client.id, - groupKeys: parseGroupKeys(groupKeys[streamId]) + groupKeys: [...parseGroupKeys(groupKeys[streamId]).entries()] }), { cacheKey([maybeStreamId]) { const { streamId } = validateOptions(maybeStreamId) @@ -420,19 +432,24 @@ export function SubscriberKeyExchange(client, { groupKeys = {} } = {}) { } }) - let sub - - async function requestKeys({ streamId, publisherId, groupKeyIds }) { + let sub: Subscription | undefined + let requestKeysStep: () => Promise + async function requestKeys({ streamId, publisherId, groupKeyIds }: { + streamId: string, + publisherId: string, + groupKeyIds: GroupKeyId[] + }) { let done = false const requestId = uuid('GroupKeyRequest') const rsaPublicKey = encryptionUtil.getPublicKey() const keyExchangeStreamId = getKeyExchangeStreamId(publisherId) - let responseTask - let cancelTask - let receivedGroupKeys = [] - let response + let responseTask: ReturnType + let cancelTask: ReturnType + let receivedGroupKeys: GroupKey[] = [] + let response: any const step = Scaffold([ async () => { + if (!sub) { throw new Error('no subscription') } cancelTask = Defer() responseTask = waitForSubMessage(sub, (content, streamMessage) => { const { messageType } = streamMessage @@ -451,7 +468,7 @@ export function SubscriberKeyExchange(client, { groupKeys = {} } = {}) { cancelTask.then(responseTask.resolve).catch(responseTask.reject) return () => { - cancelTask.resolve() + cancelTask.resolve(undefined) } }, async () => { const msg = new GroupKeyRequest({ @@ -473,16 +490,16 @@ export function SubscriberKeyExchange(client, { groupKeys = {} } = {}) { receivedGroupKeys = [] } }, - ], () => enabled && !done, { + ], async () => enabled && !done, { id: `requestKeys.${requestId}`, onChange(isGoingUp) { if (!isGoingUp && cancelTask) { - cancelTask.resolve() + cancelTask.resolve(undefined) } } }) - requestKeys.step = step + requestKeysStep = step await step() const keys = receivedGroupKeys.slice() done = true @@ -491,10 +508,10 @@ export function SubscriberKeyExchange(client, { groupKeys = {} } = {}) { } const pending = new Map() - const getBuffer = mem(() => []) - const timeouts = {} + const getBuffer = mem<(groupKeyId: GroupKeyId) => GroupKeyId[], [string]>(() => []) + const timeouts: Record> = Object.create(null) - async function getKey(streamMessage) { + async function getKey(streamMessage: StreamMessage) { const streamId = streamMessage.getStreamId() const publisherId = streamMessage.getPublisherId() const { groupKeyId } = streamMessage @@ -569,17 +586,17 @@ export function SubscriberKeyExchange(client, { groupKeys = {} } = {}) { await cancelTask } } - ], () => enabled, { + ], async () => enabled, { id: `SubscriberKeyExhangeSubscription.${client.id}`, async onDone() { // clean up requestKey - if (requestKeys.step) { - await requestKeys.step() + if (requestKeysStep) { + await requestKeysStep() } } }) - async function getGroupKey(streamMessage) { + async function getGroupKey(streamMessage: StreamMessage) { if (!streamMessage.groupKeyId) { return [] } await next() if (!enabled) { return [] } @@ -592,11 +609,11 @@ export function SubscriberKeyExchange(client, { groupKeys = {} } = {}) { enabled = true return next() }, - addNewKey(streamMessage) { + async addNewKey(streamMessage) { if (!streamMessage.newGroupKey) { return } const streamId = streamMessage.getStreamId() - const groupKeyStore = getGroupKeyStore(streamId) - groupKeyStore.add(streamMessage.newGroupKey) + const groupKeyStore = await getGroupKeyStore(streamId) + return groupKeyStore.add(streamMessage.newGroupKey) }, async stop() { enabled = false From 259d6d189cd2083ed29a9a797629a94b83b6adc6 Mon Sep 17 00:00:00 2001 From: Tim Oxley Date: Tue, 4 May 2021 14:23:51 -0400 Subject: [PATCH 12/28] refactor(keyexchange): Make group key store async on top of level-party. --- package-lock.json | 1331 ++++++++++++++++++++++++++------- package.json | 7 +- src/publish/Encrypt.ts | 12 +- src/stream/KeyExchange.ts | 144 ++-- src/stream/PersistentStore.ts | 128 +++- src/utils/index.ts | 40 + 6 files changed, 1271 insertions(+), 391 deletions(-) diff --git a/package-lock.json b/package-lock.json index d3889b407..2fb69011c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -3012,6 +3012,11 @@ "integrity": "sha512-dgasobK/Y0wVMswcipr3k0HpevxFJLijN03A8mYfEPvWvOs14v0ZlYTR4kIgMx8g4+fTyTFv8/jLCIfRqLDJ4A==", "dev": true }, + "@types/abstract-leveldown": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/@types/abstract-leveldown/-/abstract-leveldown-5.0.1.tgz", + "integrity": "sha512-wYxU3kp5zItbxKmeRYCEplS2MW7DzyBnxPGj+GJVHZEUZiK/nn5Ei1sUFgURDh+X051+zsGe28iud3oHjrYWQQ==" + }, "@types/asn1js": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/@types/asn1js/-/asn1js-2.0.0.tgz", @@ -3082,6 +3087,15 @@ "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.5.tgz", "integrity": "sha512-Q1y515GcOdTHgagaVFhHnIFQ38ygs/kmxdNpvpou+raI9UO3YZcHDngBSYKQklcKlvA7iuQlmIKbzvmxcOE9CQ==" }, + "@types/encoding-down": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@types/encoding-down/-/encoding-down-5.0.0.tgz", + "integrity": "sha512-G0MlS/+/U2RIQLcSEhhAcoMrXw3hXUCFSKbhbeEljoKMra2kq+NPX6tfOveSWQLX2hJXBo+YrvKgAGe+tFL1Aw==", + "requires": { + "@types/abstract-leveldown": "*", + "@types/level-codec": "*" + } + }, "@types/eslint": { "version": "7.2.10", "resolved": "https://registry.npmjs.org/@types/eslint/-/eslint-7.2.10.tgz", @@ -3186,6 +3200,30 @@ "integrity": "sha1-7ihweulOEdK4J7y+UnC86n8+ce4=", "dev": true }, + "@types/level": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/@types/level/-/level-6.0.0.tgz", + "integrity": "sha512-NjaUpukKfCvnV4Wk0jUaodFi2/66HxgpYghc2aV8iP+zk2NMt/9ps1eVlifqOU/+eLzMlDIY69NWkbPaAstukQ==", + "requires": { + "@types/abstract-leveldown": "*", + "@types/encoding-down": "*", + "@types/levelup": "*" + } + }, + "@types/level-codec": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/@types/level-codec/-/level-codec-9.0.0.tgz", + "integrity": "sha512-SWYkVJylo1dqblkhrr7UtmsQh4wdZA9bV1y3QJSywMPSqGfW0p1w37N1EayZtKbg1dGReIIQEEOtxk4wZvGrWQ==" + }, + "@types/levelup": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/@types/levelup/-/levelup-4.3.1.tgz", + "integrity": "sha512-n//PeTpbHLjMLTIgW5B/g06W/6iuTBHuvUka2nFL9APMSVMNe2r4enADfu3CIE9IyV9E+uquf9OEQQqrDeg24A==", + "requires": { + "@types/abstract-leveldown": "*", + "@types/node": "*" + } + }, "@types/lodash": { "version": "4.14.168", "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.168.tgz", @@ -3243,8 +3281,7 @@ "@types/node": { "version": "14.14.43", "resolved": "https://registry.npmjs.org/@types/node/-/node-14.14.43.tgz", - "integrity": "sha512-3pwDJjp1PWacPTpH0LcfhgjvurQvrZFBrC6xxjaUEZ7ifUtT32jtjPxEMMblpqd2Mvx+k8haqQJLQxolyGN/cQ==", - "dev": true + "integrity": "sha512-3pwDJjp1PWacPTpH0LcfhgjvurQvrZFBrC6xxjaUEZ7ifUtT32jtjPxEMMblpqd2Mvx+k8haqQJLQxolyGN/cQ==" }, "@types/node-fetch": { "version": "2.5.10", @@ -3670,6 +3707,34 @@ "integrity": "sha512-9IK9EadsbHo6jLWIpxpR6pL0sazTXV6+SQv25ZB+F7Bj9mJNaOc4nCRabwd5M/JwmUa8idz6Eci6eKfJryPs6Q==", "dev": true }, + "abbrev": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", + "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==" + }, + "abstract-leveldown": { + "version": "6.2.3", + "resolved": "https://registry.npmjs.org/abstract-leveldown/-/abstract-leveldown-6.2.3.tgz", + "integrity": "sha512-BsLm5vFMRUrrLeCcRc+G0t2qOaTzpoJQLOubq2XM72eNpjF5UdU5o/5NvlNhx95XHcAvcl8OMXr4mlg/fRgUXQ==", + "requires": { + "buffer": "^5.5.0", + "immediate": "^3.2.3", + "level-concat-iterator": "~2.0.0", + "level-supports": "~1.0.0", + "xtend": "~4.0.0" + }, + "dependencies": { + "buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "requires": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + } + } + }, "accepts": { "version": "1.3.7", "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.7.tgz", @@ -3742,7 +3807,6 @@ "version": "6.12.6", "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", - "dev": true, "requires": { "fast-deep-equal": "^3.1.1", "fast-json-stable-stringify": "^2.0.0", @@ -3750,32 +3814,6 @@ "uri-js": "^4.2.2" } }, - "ajv-formats": { - "version": "1.6.1", - "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-1.6.1.tgz", - "integrity": "sha512-4CjkH20If1lhR5CGtqkrVg3bbOtFEG80X9v6jDOIUhbzzbB+UzPBGy8GQhUNVZ0yvMHdMpawCOcy5ydGMsagGQ==", - "requires": { - "ajv": "^7.0.0" - }, - "dependencies": { - "ajv": { - "version": "7.2.4", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-7.2.4.tgz", - "integrity": "sha512-nBeQgg/ZZA3u3SYxyaDvpvDtgZ/EZPF547ARgZBrG9Bhu1vKDwAIjtIf+sDtJUKa2zOcEbmRLBRSyMraS/Oy1A==", - "requires": { - "fast-deep-equal": "^3.1.1", - "json-schema-traverse": "^1.0.0", - "require-from-string": "^2.0.2", - "uri-js": "^4.2.2" - } - }, - "json-schema-traverse": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", - "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==" - } - } - }, "ajv-keywords": { "version": "3.5.2", "resolved": "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-3.5.2.tgz", @@ -3841,6 +3879,49 @@ } } }, + "aproba": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/aproba/-/aproba-1.2.0.tgz", + "integrity": "sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw==" + }, + "are-we-there-yet": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-1.1.5.tgz", + "integrity": "sha512-5hYdAkZlcG8tOLujVDTgCT+uPX0VnpAH28gWsLfzpXYm7wP6mp5Q/gYyR7YQ0cKVJcXJnl3j2kpBan13PtQf6w==", + "requires": { + "delegates": "^1.0.0", + "readable-stream": "^2.0.6" + }, + "dependencies": { + "readable-stream": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", + "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, + "string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "requires": { + "safe-buffer": "~5.1.0" + } + } + } + }, "argparse": { "version": "1.0.10", "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", @@ -3932,7 +4013,6 @@ "version": "0.2.4", "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.4.tgz", "integrity": "sha512-jxwzQpLQjSmWXgwaCZE9Nz+glAG01yF1QnWgbhGwHI5A6FRIEY6IVqtHhIepHqI7/kyEyQEagBC5mBEFlIYvdg==", - "dev": true, "requires": { "safer-buffer": "~2.1.0" } @@ -3967,8 +4047,7 @@ "assert-plus": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", - "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=", - "dev": true + "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=" }, "assertion-error": { "version": "1.1.0", @@ -4013,8 +4092,7 @@ "asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", - "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=", - "dev": true + "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=" }, "at-least-node": { "version": "1.0.0", @@ -4028,11 +4106,6 @@ "integrity": "sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg==", "dev": true }, - "atomically": { - "version": "1.7.0", - "resolved": "https://registry.npmjs.org/atomically/-/atomically-1.7.0.tgz", - "integrity": "sha512-Xcz9l0z7y9yQ9rdDaxlmaI4uJHf/T8g9hOEzJcsEqX2SjCj4J20uK7+ldkDHMbpJDK76wF7xEIgxc/vSlsfw5w==" - }, "available-typed-arrays": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/available-typed-arrays/-/available-typed-arrays-1.0.2.tgz", @@ -4045,14 +4118,12 @@ "aws-sign2": { "version": "0.7.0", "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz", - "integrity": "sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg=", - "dev": true + "integrity": "sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg=" }, "aws4": { "version": "1.11.0", "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.11.0.tgz", - "integrity": "sha512-xh1Rl34h6Fi1DC2WWKfxUTVqRsNnr6LsKz2+hfwDxQJWmrx8+c7ylaqBMcHfl1U1r2dsifOvKX3LQuLNZ+XSvA==", - "dev": true + "integrity": "sha512-xh1Rl34h6Fi1DC2WWKfxUTVqRsNnr6LsKz2+hfwDxQJWmrx8+c7ylaqBMcHfl1U1r2dsifOvKX3LQuLNZ+XSvA==" }, "axios": { "version": "0.21.1", @@ -4486,8 +4557,7 @@ "balanced-match": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", - "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=", - "dev": true + "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=" }, "base": { "version": "0.11.2", @@ -4553,7 +4623,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", "integrity": "sha1-pDAdOJtqQ/m2f/PKEaP2Y342Dp4=", - "dev": true, "requires": { "tweetnacl": "^0.14.3" } @@ -4586,6 +4655,15 @@ "dev": true, "optional": true }, + "block-stream": { + "version": "0.0.9", + "resolved": "https://registry.npmjs.org/block-stream/-/block-stream-0.0.9.tgz", + "integrity": "sha1-E+v+d4oDIFz+A3UUgeu0szAMEmo=", + "optional": true, + "requires": { + "inherits": "~2.0.0" + } + }, "bluebird": { "version": "3.7.2", "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz", @@ -4642,7 +4720,6 @@ "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", - "dev": true, "requires": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -4928,8 +5005,12 @@ "caseless": { "version": "0.12.0", "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz", - "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=", - "dev": true + "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=" + }, + "catering": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/catering/-/catering-2.0.0.tgz", + "integrity": "sha512-aD/WmxhGwUGsVPrj8C80vH7C7GphJilYVSdudoV4u16XdrLF7CVyfBmENsc4tLTVsJJzCRid8GbwJ7mcPLee6Q==" }, "chai-nightwatch": { "version": "0.4.0", @@ -5209,6 +5290,11 @@ "integrity": "sha1-bqa989hTrlTMuOR7+gvz+QMfsYQ=", "dev": true }, + "code-point-at": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz", + "integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c=" + }, "collect-v8-coverage": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/collect-v8-coverage/-/collect-v8-coverage-1.0.1.tgz", @@ -5256,7 +5342,6 @@ "version": "1.0.8", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", - "dev": true, "requires": { "delayed-stream": "~1.0.0" } @@ -5303,75 +5388,7 @@ "concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", - "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=", - "dev": true - }, - "conf": { - "version": "9.0.2", - "resolved": "https://registry.npmjs.org/conf/-/conf-9.0.2.tgz", - "integrity": "sha512-rLSiilO85qHgaTBIIHQpsv8z+NnVfZq3cKuYNCXN1AOqPzced0GWZEe/A517VldRLyQYXUMyV+vszavE2jSAqw==", - "requires": { - "ajv": "^7.0.3", - "ajv-formats": "^1.5.1", - "atomically": "^1.7.0", - "debounce-fn": "^4.0.0", - "dot-prop": "^6.0.1", - "env-paths": "^2.2.0", - "json-schema-typed": "^7.0.3", - "make-dir": "^3.1.0", - "onetime": "^5.1.2", - "pkg-up": "^3.1.0", - "semver": "^7.3.4" - }, - "dependencies": { - "ajv": { - "version": "7.2.4", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-7.2.4.tgz", - "integrity": "sha512-nBeQgg/ZZA3u3SYxyaDvpvDtgZ/EZPF547ARgZBrG9Bhu1vKDwAIjtIf+sDtJUKa2zOcEbmRLBRSyMraS/Oy1A==", - "requires": { - "fast-deep-equal": "^3.1.1", - "json-schema-traverse": "^1.0.0", - "require-from-string": "^2.0.2", - "uri-js": "^4.2.2" - } - }, - "dot-prop": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-6.0.1.tgz", - "integrity": "sha512-tE7ztYzXHIeyvc7N+hR3oi7FIbf/NIjVP9hmAt3yMXzrQ072/fpjGLx2GxNxGxUl5V73MEqYzioOMoVhGMJ5cA==", - "requires": { - "is-obj": "^2.0.0" - } - }, - "json-schema-traverse": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", - "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==" - }, - "make-dir": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-3.1.0.tgz", - "integrity": "sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw==", - "requires": { - "semver": "^6.0.0" - }, - "dependencies": { - "semver": { - "version": "6.3.0", - "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", - "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==" - } - } - }, - "semver": { - "version": "7.3.5", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.5.tgz", - "integrity": "sha512-PoeGJYh8HK4BTO/a9Tf6ZG3veo/A7ZVsYrSA6J8ny9nb3B1VrpkuN+z9OE5wfE5p6H4LchYZsegiQgbJD94ZFQ==", - "requires": { - "lru-cache": "^6.0.0" - } - } - } + "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=" }, "confusing-browser-globals": { "version": "1.0.10", @@ -5379,6 +5396,11 @@ "integrity": "sha512-gNld/3lySHwuhaVluJUKLePYirM3QNCKzVxqAdhJII9/WXKVX5PURzMVJspS1jTslSqjeuG4KMVTSouit5YPHA==", "dev": true }, + "console-control-strings": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz", + "integrity": "sha1-PXz0Rk22RG6mRL9LOVB/mFEAjo4=" + }, "contains-path": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/contains-path/-/contains-path-0.1.0.tgz", @@ -5511,8 +5533,7 @@ "core-util-is": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", - "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=", - "dev": true + "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" }, "cosmiconfig": { "version": "7.0.0", @@ -5650,7 +5671,6 @@ "version": "1.14.1", "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz", "integrity": "sha1-hTz6D3y+L+1d4gMmuN1YEDX24vA=", - "dev": true, "requires": { "assert-plus": "^1.0.0" } @@ -5672,14 +5692,6 @@ "whatwg-url": "^8.0.0" } }, - "debounce-fn": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/debounce-fn/-/debounce-fn-4.0.0.tgz", - "integrity": "sha512-8pYCQiL9Xdcg0UPSD3d+0KMlOjp+KGU5EPwYddgzQ7DATsg4fuUDjQtsYLmWjnk2obnNHgV3vE2Y4jejSOJVBQ==", - "requires": { - "mimic-fn": "^3.0.0" - } - }, "debug": { "version": "4.3.2", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.2.tgz", @@ -5747,6 +5759,11 @@ } } }, + "deep-extend": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", + "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==" + }, "deep-is": { "version": "0.1.3", "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.3.tgz", @@ -5768,6 +5785,15 @@ "clone": "^1.0.2" } }, + "deferred-leveldown": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/deferred-leveldown/-/deferred-leveldown-5.3.0.tgz", + "integrity": "sha512-a59VOT+oDy7vtAbLRCZwWgxu2BaCfd5Hk7wxJd48ei7I+nsg8Orlb9CLG0PMZienk9BSUKgeAqkO2+Lw+1+Ukw==", + "requires": { + "abstract-leveldown": "~6.2.1", + "inherits": "^2.0.3" + } + }, "define-properties": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.3.tgz", @@ -5856,8 +5882,12 @@ "delayed-stream": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", - "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=", - "dev": true + "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=" + }, + "delegates": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz", + "integrity": "sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o=" }, "depd": { "version": "1.1.2", @@ -5881,6 +5911,11 @@ "integrity": "sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA=", "dev": true }, + "detect-libc": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-1.0.3.tgz", + "integrity": "sha1-+hN8S9aY7fVc1c0CrFWfkaTEups=" + }, "detect-newline": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/detect-newline/-/detect-newline-3.1.0.tgz", @@ -5998,11 +6033,21 @@ } } }, + "duplexify": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-4.1.1.tgz", + "integrity": "sha512-DY3xVEmVHTv1wSzKNbwoU6nVjzI369Y6sPoqfYr0/xlx3IdX2n94xIszTcjPO8W8ZIv0Wb0PXNcjuZyT4wiICA==", + "requires": { + "end-of-stream": "^1.4.1", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1", + "stream-shift": "^1.0.0" + } + }, "ecc-jsbn": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz", "integrity": "sha1-OoOpBOVDUyh4dMVkt1SThoSamMk=", - "dev": true, "requires": { "jsbn": "~0.1.0", "safer-buffer": "^2.1.0" @@ -6067,11 +6112,21 @@ "integrity": "sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k=", "dev": true }, + "encoding-down": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/encoding-down/-/encoding-down-6.3.0.tgz", + "integrity": "sha512-QKrV0iKR6MZVJV08QY0wp1e7vF6QbhnbQhb07bwpEyuz4uZiZgPlEGdkCROuFkUwdxlFaiPIhjyarH1ee/3vhw==", + "requires": { + "abstract-leveldown": "^6.2.1", + "inherits": "^2.0.3", + "level-codec": "^9.0.0", + "level-errors": "^2.0.0" + } + }, "end-of-stream": { "version": "1.4.4", "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", - "dev": true, "requires": { "once": "^1.4.0" } @@ -6111,7 +6166,6 @@ "version": "0.1.8", "resolved": "https://registry.npmjs.org/errno/-/errno-0.1.8.tgz", "integrity": "sha512-dJ6oBr5SQ1VSd9qkk7ByRgb/1SH4JZjCHSW/mr63/QcXO9zLVxvJ6Oy13nio03rxpSnVDDjFor75SjVeZWPW/A==", - "dev": true, "requires": { "prr": "~1.0.1" } @@ -7025,8 +7079,7 @@ "extend": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", - "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==", - "dev": true + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==" }, "extend-shallow": { "version": "3.0.2", @@ -7129,8 +7182,7 @@ "extsprintf": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz", - "integrity": "sha1-lpGEQOMEGnpBT4xS48V06zw+HgU=", - "dev": true + "integrity": "sha1-lpGEQOMEGnpBT4xS48V06zw+HgU=" }, "fast-deep-equal": { "version": "3.1.3", @@ -7208,8 +7260,7 @@ "fast-json-stable-stringify": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", - "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", - "dev": true + "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==" }, "fast-levenshtein": { "version": "2.0.6", @@ -7421,14 +7472,12 @@ "forever-agent": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz", - "integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE=", - "dev": true + "integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE=" }, "form-data": { "version": "2.3.3", "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz", "integrity": "sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==", - "dev": true, "requires": { "asynckit": "^0.4.0", "combined-stream": "^1.0.6", @@ -7485,8 +7534,7 @@ "fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", - "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=", - "dev": true + "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=" }, "fsevents": { "version": "2.3.2", @@ -7495,6 +7543,38 @@ "dev": true, "optional": true }, + "fstream": { + "version": "1.0.12", + "resolved": "https://registry.npmjs.org/fstream/-/fstream-1.0.12.tgz", + "integrity": "sha512-WvJ193OHa0GHPEL+AycEJgxvBEwyfRkN1vhjca23OaPVMCaLCXTd5qAu82AjTcgP1UJmytkOKb63Ypde7raDIg==", + "optional": true, + "requires": { + "graceful-fs": "^4.1.2", + "inherits": "~2.0.0", + "mkdirp": ">=0.5 0", + "rimraf": "2" + }, + "dependencies": { + "mkdirp": { + "version": "0.5.5", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz", + "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==", + "optional": true, + "requires": { + "minimist": "^1.2.5" + } + }, + "rimraf": { + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz", + "integrity": "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==", + "optional": true, + "requires": { + "glob": "^7.1.3" + } + } + } + }, "ftp": { "version": "0.3.10", "resolved": "https://registry.npmjs.org/ftp/-/ftp-0.3.10.tgz", @@ -7542,6 +7622,41 @@ "integrity": "sha1-GwqzvVU7Kg1jmdKcDj6gslIHgyc=", "dev": true }, + "gauge": { + "version": "2.7.4", + "resolved": "https://registry.npmjs.org/gauge/-/gauge-2.7.4.tgz", + "integrity": "sha1-LANAXHU4w51+s3sxcCLjJfsBi/c=", + "requires": { + "aproba": "^1.0.3", + "console-control-strings": "^1.0.0", + "has-unicode": "^2.0.0", + "object-assign": "^4.1.0", + "signal-exit": "^3.0.0", + "string-width": "^1.0.1", + "strip-ansi": "^3.0.1", + "wide-align": "^1.1.0" + }, + "dependencies": { + "is-fullwidth-code-point": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz", + "integrity": "sha1-754xOG8DGn8NZDr4L95QxFfvAMs=", + "requires": { + "number-is-nan": "^1.0.0" + } + }, + "string-width": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", + "integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=", + "requires": { + "code-point-at": "^1.0.0", + "is-fullwidth-code-point": "^1.0.0", + "strip-ansi": "^3.0.0" + } + } + } + }, "geckodriver": { "version": "1.22.3", "resolved": "https://registry.npmjs.org/geckodriver/-/geckodriver-1.22.3.tgz", @@ -7622,7 +7737,6 @@ "version": "0.1.7", "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz", "integrity": "sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo=", - "dev": true, "requires": { "assert-plus": "^1.0.0" } @@ -7650,7 +7764,6 @@ "version": "7.1.6", "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz", "integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==", - "dev": true, "requires": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", @@ -7785,8 +7898,7 @@ "graceful-fs": { "version": "4.2.6", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.6.tgz", - "integrity": "sha512-nTnJ528pbqxYanhpDYsi4Rd8MAeaBA67+RZ10CM1m3bTAVFEDcd5AuA4a6W5YkGZ1iNXHzZz8T6TBKLeBuNriQ==", - "dev": true + "integrity": "sha512-nTnJ528pbqxYanhpDYsi4Rd8MAeaBA67+RZ10CM1m3bTAVFEDcd5AuA4a6W5YkGZ1iNXHzZz8T6TBKLeBuNriQ==" }, "growl": { "version": "1.10.5", @@ -7834,14 +7946,12 @@ "har-schema": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz", - "integrity": "sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI=", - "dev": true + "integrity": "sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI=" }, "har-validator": { "version": "5.1.5", "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.5.tgz", "integrity": "sha512-nmT2T0lljbxdQZfspsno9hgrG3Uir6Ks5afism62poxqBM6sDnMEuPmzTq8XN0OEwqKLLdh1jQI3qyE66Nzb3w==", - "dev": true, "requires": { "ajv": "^6.12.3", "har-schema": "^2.0.0" @@ -7889,6 +7999,11 @@ "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.1.tgz", "integrity": "sha512-PLcsoqu++dmEIZB+6totNFKq/7Do+Z0u4oT0zKOJNl3lYK6vGwwu2hjHs+68OEZbTjiUE9bgOABXbP/GvrS0Kg==" }, + "has-unicode": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz", + "integrity": "sha1-4Ob+aijPUROIVeCG0Wkedx3iqLk=" + }, "has-value": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/has-value/-/has-value-1.0.0.tgz", @@ -8013,7 +8128,6 @@ "version": "1.2.0", "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz", "integrity": "sha1-muzZJRFHcvPZW2WmCruPfBj7rOE=", - "dev": true, "requires": { "assert-plus": "^1.0.0", "jsprim": "^1.2.2", @@ -8046,7 +8160,6 @@ "version": "0.4.24", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", - "dev": true, "requires": { "safer-buffer": ">= 2.1.2 < 3" } @@ -8062,6 +8175,19 @@ "integrity": "sha512-BMpfD7PpiETpBl/A6S498BaIJ6Y/ABT93ETbby2fP00v4EbvPBXWEoaR1UBPKs3iR53pJY7EtZk5KACI57i1Uw==", "dev": true }, + "ignore-walk": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/ignore-walk/-/ignore-walk-3.0.3.tgz", + "integrity": "sha512-m7o6xuOaT1aqheYHKf8W6J5pYH85ZI9w077erOzLje3JsB1gkafkAhHHY19dqjulgIZHFm32Cp5uNZgcQqdJKw==", + "requires": { + "minimatch": "^3.0.4" + } + }, + "immediate": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/immediate/-/immediate-3.3.0.tgz", + "integrity": "sha512-HR7EVodfFUdQCTIeySw+WDRFJlPcLOJbXfwwZ7Oom6tjsvZ3bOkCDJHehQC3nxJrv7+f9XecwazynjU8e4Vw3Q==" + }, "import-fresh": { "version": "3.3.0", "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.0.tgz", @@ -8106,7 +8232,6 @@ "version": "1.0.6", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", - "dev": true, "requires": { "once": "^1.3.0", "wrappy": "1" @@ -8120,8 +8245,7 @@ "ini": { "version": "1.3.8", "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", - "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==", - "dev": true + "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==" }, "interpret": { "version": "1.4.0", @@ -8357,7 +8481,8 @@ "is-obj": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/is-obj/-/is-obj-2.0.0.tgz", - "integrity": "sha512-drqDG3cbczxxEJRoOXcOjtdp1J/lyp1mNn0xaznRs8+muBhgQcrnbspox5X5fOw0HnMnbfDzvnEMEtqDEJEo8w==" + "integrity": "sha512-drqDG3cbczxxEJRoOXcOjtdp1J/lyp1mNn0xaznRs8+muBhgQcrnbspox5X5fOw0HnMnbfDzvnEMEtqDEJEo8w==", + "dev": true }, "is-path-cwd": { "version": "2.2.0", @@ -8460,8 +8585,7 @@ "is-typedarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", - "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=", - "dev": true + "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=" }, "is-url": { "version": "1.2.4", @@ -8499,14 +8623,12 @@ "isarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", - "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=", - "dev": true + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" }, "isexe": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", - "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=", - "dev": true + "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=" }, "isobject": { "version": "3.0.1", @@ -8517,8 +8639,7 @@ "isstream": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", - "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=", - "dev": true + "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=" }, "istanbul-lib-coverage": { "version": "3.0.0", @@ -10190,8 +10311,7 @@ "jsbn": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz", - "integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM=", - "dev": true + "integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM=" }, "jsdom": { "version": "16.4.0", @@ -10248,19 +10368,12 @@ "json-schema": { "version": "0.2.3", "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz", - "integrity": "sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM=", - "dev": true + "integrity": "sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM=" }, "json-schema-traverse": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", - "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", - "dev": true - }, - "json-schema-typed": { - "version": "7.0.3", - "resolved": "https://registry.npmjs.org/json-schema-typed/-/json-schema-typed-7.0.3.tgz", - "integrity": "sha512-7DE8mpG+/fVw+dTpjbxnx47TaMnDfOI1jwft9g1VybltZCduyRQPJPvc+zzKY9WPHxhPWczyFuYa6I8Mw4iU5A==" + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==" }, "json-stable-stringify-without-jsonify": { "version": "1.0.1", @@ -10271,8 +10384,7 @@ "json-stringify-safe": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", - "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=", - "dev": true + "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=" }, "json5": { "version": "2.2.0", @@ -10302,7 +10414,6 @@ "version": "1.4.1", "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz", "integrity": "sha1-MT5mvB5cwG5Di8G3SZwuXFastqI=", - "dev": true, "requires": { "assert-plus": "1.0.0", "extsprintf": "1.3.0", @@ -10328,14 +10439,360 @@ "integrity": "sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w==", "dev": true }, - "leven": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/leven/-/leven-3.1.0.tgz", - "integrity": "sha512-qsda+H8jTaUaN/x5vzW2rzc+8Rw4TAQ/4KjB46IwK5VH+IlVeeeje/EoZRpiXvIqjFgK84QffqPztGI3VBLG1A==", - "dev": true - }, - "levn": { - "version": "0.4.1", + "length-prefixed-stream": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/length-prefixed-stream/-/length-prefixed-stream-2.0.0.tgz", + "integrity": "sha512-dvjTuWTKWe0oEznQcG6a9osfiYknCs7DEFJMP88n9Y581IFhYh1sZIgAFcuDOojKB0G7ftPreKhh4D0kh/VPjQ==", + "requires": { + "inherits": "^2.0.3", + "readable-stream": "^3.1.1", + "varint": "^5.0.0" + } + }, + "level": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/level/-/level-7.0.0.tgz", + "integrity": "sha512-QrBnjcWywalh86ms9hfizvxT5aBHrgWEu6rLChS9tFE2wwFU3aI1r0v+2SSZIyeUr4O4PFo8+sCc1kebahdhlw==", + "requires": { + "level-js": "^6.0.0", + "level-packager": "^6.0.0", + "leveldown": "^6.0.0" + } + }, + "level-codec": { + "version": "9.0.2", + "resolved": "https://registry.npmjs.org/level-codec/-/level-codec-9.0.2.tgz", + "integrity": "sha512-UyIwNb1lJBChJnGfjmO0OR+ezh2iVu1Kas3nvBS/BzGnx79dv6g7unpKIDNPMhfdTEGoc7mC8uAu51XEtX+FHQ==", + "requires": { + "buffer": "^5.6.0" + }, + "dependencies": { + "buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "requires": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + } + } + }, + "level-concat-iterator": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/level-concat-iterator/-/level-concat-iterator-2.0.1.tgz", + "integrity": "sha512-OTKKOqeav2QWcERMJR7IS9CUo1sHnke2C0gkSmcR7QuEtFNLLzHQAvnMw8ykvEcv0Qtkg0p7FOwP1v9e5Smdcw==" + }, + "level-errors": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/level-errors/-/level-errors-2.0.1.tgz", + "integrity": "sha512-UVprBJXite4gPS+3VznfgDSU8PTRuVX0NXwoWW50KLxd2yw4Y1t2JUR5In1itQnudZqRMT9DlAM3Q//9NCjCFw==", + "requires": { + "errno": "~0.1.1" + } + }, + "level-iterator-stream": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/level-iterator-stream/-/level-iterator-stream-4.0.2.tgz", + "integrity": "sha512-ZSthfEqzGSOMWoUGhTXdX9jv26d32XJuHz/5YnuHZzH6wldfWMOVwI9TBtKcya4BKTyTt3XVA0A3cF3q5CY30Q==", + "requires": { + "inherits": "^2.0.4", + "readable-stream": "^3.4.0", + "xtend": "^4.0.2" + } + }, + "level-js": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/level-js/-/level-js-6.0.0.tgz", + "integrity": "sha512-7dp7JuaoQoqKW4ZGvrV1RB5f51/ktLdEo9fSDsh3Ofmg7sKCMu3X0CIngbY/IUz/YyskhN7LRvEVIkZHCY3LKQ==", + "requires": { + "abstract-leveldown": "^7.0.0", + "buffer": "^6.0.3", + "inherits": "^2.0.3", + "ltgt": "^2.1.2" + }, + "dependencies": { + "abstract-leveldown": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/abstract-leveldown/-/abstract-leveldown-7.0.0.tgz", + "integrity": "sha512-mFAi5sB/UjpNYglrQ4irzdmr2mbQtE94OJbrAYuK2yRARjH/OACinN1meOAorfnaLPMQdFymSQMlkiDm9AXXKQ==", + "requires": { + "buffer": "^6.0.3", + "is-buffer": "^2.0.5", + "level-concat-iterator": "^3.0.0", + "level-supports": "^2.0.0", + "queue-microtask": "^1.2.3" + } + }, + "is-buffer": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-2.0.5.tgz", + "integrity": "sha512-i2R6zNFDwgEHJyQUtJEk0XFi1i0dPFn/oqjK3/vPCcDeJvW5NQ83V8QbicfF1SupOaB0h8ntgBC2YiE7dfyctQ==" + }, + "level-concat-iterator": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/level-concat-iterator/-/level-concat-iterator-3.0.0.tgz", + "integrity": "sha512-UHGiIdj+uiFQorOrURRvJF3Ei0uHc89ciM/aRi0qsWDV2f0HXypeXUPhJKL6DsONgSR76Pc0AI4sKYEYYRn2Dg==" + }, + "level-supports": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/level-supports/-/level-supports-2.0.0.tgz", + "integrity": "sha512-8UJgzo1pvWP1wq80ZlkL19fPeK7tlyy0sBY90+2pj0x/kvzHCoLDWyuFJJMrsTn33oc7hbMkS3SkjCxMRPHWaw==" + }, + "queue-microtask": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", + "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==" + } + } + }, + "level-packager": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/level-packager/-/level-packager-6.0.0.tgz", + "integrity": "sha512-me656XRWfOVqs9wc+mWckZ6Rb1GuP33ndN4ZntDXwXFspX8cGA++Y+YqJsdE/mjTiipTuxXf047Z4rV62nOVuw==", + "requires": { + "encoding-down": "^7.0.0", + "levelup": "^5.0.0" + }, + "dependencies": { + "abstract-leveldown": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/abstract-leveldown/-/abstract-leveldown-7.0.0.tgz", + "integrity": "sha512-mFAi5sB/UjpNYglrQ4irzdmr2mbQtE94OJbrAYuK2yRARjH/OACinN1meOAorfnaLPMQdFymSQMlkiDm9AXXKQ==", + "requires": { + "buffer": "^6.0.3", + "is-buffer": "^2.0.5", + "level-concat-iterator": "^3.0.0", + "level-supports": "^2.0.0", + "queue-microtask": "^1.2.3" + } + }, + "deferred-leveldown": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/deferred-leveldown/-/deferred-leveldown-6.0.0.tgz", + "integrity": "sha512-F6CLAZzNeURojlH4MCigZr54tNz+xDSi06YXsDr5uLSKeF3JKnvnQWTqd+RETh2hbWTJR3qDzGicQOWS5ZQ1BQ==", + "requires": { + "abstract-leveldown": "^7.0.0", + "inherits": "^2.0.3" + } + }, + "encoding-down": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/encoding-down/-/encoding-down-7.0.0.tgz", + "integrity": "sha512-hor6z2W/ZrVqDYMawQp7VtfEt6BrvYw+mgjWLauUMZsIBjMt1/k5aa+JreLbtjwJdkjrZ39TU+pV5GpHPGRpog==", + "requires": { + "abstract-leveldown": "^7.0.0", + "inherits": "^2.0.3", + "level-codec": "^10.0.0", + "level-errors": "^3.0.0" + } + }, + "errno": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/errno/-/errno-1.0.0.tgz", + "integrity": "sha512-3zV5mFS1E8/1bPxt/B0xxzI1snsg3uSCIh6Zo1qKg6iMw93hzPANk9oBFzSFBFrwuVoQuE3rLoouAUfwOAj1wQ==", + "requires": { + "prr": "~1.0.1" + } + }, + "is-buffer": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-2.0.5.tgz", + "integrity": "sha512-i2R6zNFDwgEHJyQUtJEk0XFi1i0dPFn/oqjK3/vPCcDeJvW5NQ83V8QbicfF1SupOaB0h8ntgBC2YiE7dfyctQ==" + }, + "level-codec": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/level-codec/-/level-codec-10.0.0.tgz", + "integrity": "sha512-QW3VteVNAp6c/LuV6nDjg7XDXx9XHK4abmQarxZmlRSDyXYk20UdaJTSX6yzVvQ4i0JyWSB7jert0DsyD/kk6g==", + "requires": { + "buffer": "^6.0.3" + } + }, + "level-concat-iterator": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/level-concat-iterator/-/level-concat-iterator-3.0.0.tgz", + "integrity": "sha512-UHGiIdj+uiFQorOrURRvJF3Ei0uHc89ciM/aRi0qsWDV2f0HXypeXUPhJKL6DsONgSR76Pc0AI4sKYEYYRn2Dg==" + }, + "level-errors": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/level-errors/-/level-errors-3.0.0.tgz", + "integrity": "sha512-MZXOQT061uEjxxxq4C/Jf+M3RdEKK9e3NbxlN7yOp1LDYoLVAhE2i1j0b7XqXfl8FjFtUL7phwr3Sn0wXXoMqA==", + "requires": { + "errno": "^1.0.0" + } + }, + "level-iterator-stream": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/level-iterator-stream/-/level-iterator-stream-5.0.0.tgz", + "integrity": "sha512-wnb1+o+CVFUDdiSMR/ZymE2prPs3cjVLlXuDeSq9Zb8o032XrabGEXcTCsBxprAtseO3qvFeGzh6406z9sOTRA==", + "requires": { + "inherits": "^2.0.4", + "readable-stream": "^3.4.0" + } + }, + "level-supports": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/level-supports/-/level-supports-2.0.0.tgz", + "integrity": "sha512-8UJgzo1pvWP1wq80ZlkL19fPeK7tlyy0sBY90+2pj0x/kvzHCoLDWyuFJJMrsTn33oc7hbMkS3SkjCxMRPHWaw==" + }, + "levelup": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/levelup/-/levelup-5.0.0.tgz", + "integrity": "sha512-P4IKS4J17b6dzm8iI8Irv5gvOlrqJv04Lrpq1rAgZvjR2IsVSjbXQQo1LoK/PJuouxepLE8CTIiKGmHQYZnneA==", + "requires": { + "catering": "^2.0.0", + "deferred-leveldown": "^6.0.0", + "level-errors": "^3.0.0", + "level-iterator-stream": "^5.0.0", + "level-supports": "^2.0.0", + "queue-microtask": "^1.2.3" + } + }, + "queue-microtask": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", + "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==" + } + } + }, + "level-party": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/level-party/-/level-party-5.0.0.tgz", + "integrity": "sha512-wtvzpTTDv5CB/Wca+uaDmIFqMesY7NhkPLVSRtsx6DMvxpP4vvrIlRVAthLnysdpUNYcZvtSbLqRDFkwu0y+aw==", + "requires": { + "level": "^6.0.0", + "multileveldown": "^3.0.0", + "readable-stream": "^3.6.0" + }, + "dependencies": { + "buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "requires": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, + "level": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/level/-/level-6.0.1.tgz", + "integrity": "sha512-psRSqJZCsC/irNhfHzrVZbmPYXDcEYhA5TVNwr+V92jF44rbf86hqGp8fiT702FyiArScYIlPSBTDUASCVNSpw==", + "requires": { + "level-js": "^5.0.0", + "level-packager": "^5.1.0", + "leveldown": "^5.4.0" + } + }, + "level-js": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/level-js/-/level-js-5.0.2.tgz", + "integrity": "sha512-SnBIDo2pdO5VXh02ZmtAyPP6/+6YTJg2ibLtl9C34pWvmtMEmRTWpra+qO/hifkUtBTOtfx6S9vLDjBsBK4gRg==", + "requires": { + "abstract-leveldown": "~6.2.3", + "buffer": "^5.5.0", + "inherits": "^2.0.3", + "ltgt": "^2.1.2" + } + }, + "level-packager": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/level-packager/-/level-packager-5.1.1.tgz", + "integrity": "sha512-HMwMaQPlTC1IlcwT3+swhqf/NUO+ZhXVz6TY1zZIIZlIR0YSn8GtAAWmIvKjNY16ZkEg/JcpAuQskxsXqC0yOQ==", + "requires": { + "encoding-down": "^6.3.0", + "levelup": "^4.3.2" + } + }, + "leveldown": { + "version": "5.6.0", + "resolved": "https://registry.npmjs.org/leveldown/-/leveldown-5.6.0.tgz", + "integrity": "sha512-iB8O/7Db9lPaITU1aA2txU/cBEXAt4vWwKQRrrWuS6XDgbP4QZGj9BL2aNbwb002atoQ/lIotJkfyzz+ygQnUQ==", + "requires": { + "abstract-leveldown": "~6.2.1", + "napi-macros": "~2.0.0", + "node-gyp-build": "~4.1.0" + } + }, + "node-gyp-build": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.1.1.tgz", + "integrity": "sha512-dSq1xmcPDKPZ2EED2S6zw/b9NKsqzXRE6dVr8TVQnI3FJOTteUMuqF3Qqs6LZg+mLGYJWqQzMbIjMtJqTv87nQ==" + } + } + }, + "level-supports": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/level-supports/-/level-supports-1.0.1.tgz", + "integrity": "sha512-rXM7GYnW8gsl1vedTJIbzOrRv85c/2uCMpiiCzO2fndd06U/kUXEEU9evYn4zFggBOg36IsBW8LzqIpETwwQzg==", + "requires": { + "xtend": "^4.0.2" + } + }, + "leveldown": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/leveldown/-/leveldown-6.0.0.tgz", + "integrity": "sha512-NEsyqpfdDhpFO49Zm9htNSsWixMa9Q9sUXgrBTaQNPyPo2Kx1wRctgIXMzc7tduXJqNff8QAwulv2eZDboghxQ==", + "requires": { + "abstract-leveldown": "^7.0.0", + "napi-macros": "~2.0.0", + "node-gyp-build": "~4.2.1" + }, + "dependencies": { + "abstract-leveldown": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/abstract-leveldown/-/abstract-leveldown-7.0.0.tgz", + "integrity": "sha512-mFAi5sB/UjpNYglrQ4irzdmr2mbQtE94OJbrAYuK2yRARjH/OACinN1meOAorfnaLPMQdFymSQMlkiDm9AXXKQ==", + "requires": { + "buffer": "^6.0.3", + "is-buffer": "^2.0.5", + "level-concat-iterator": "^3.0.0", + "level-supports": "^2.0.0", + "queue-microtask": "^1.2.3" + } + }, + "is-buffer": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-2.0.5.tgz", + "integrity": "sha512-i2R6zNFDwgEHJyQUtJEk0XFi1i0dPFn/oqjK3/vPCcDeJvW5NQ83V8QbicfF1SupOaB0h8ntgBC2YiE7dfyctQ==" + }, + "level-concat-iterator": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/level-concat-iterator/-/level-concat-iterator-3.0.0.tgz", + "integrity": "sha512-UHGiIdj+uiFQorOrURRvJF3Ei0uHc89ciM/aRi0qsWDV2f0HXypeXUPhJKL6DsONgSR76Pc0AI4sKYEYYRn2Dg==" + }, + "level-supports": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/level-supports/-/level-supports-2.0.0.tgz", + "integrity": "sha512-8UJgzo1pvWP1wq80ZlkL19fPeK7tlyy0sBY90+2pj0x/kvzHCoLDWyuFJJMrsTn33oc7hbMkS3SkjCxMRPHWaw==" + }, + "queue-microtask": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", + "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==" + } + } + }, + "levelup": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/levelup/-/levelup-4.4.0.tgz", + "integrity": "sha512-94++VFO3qN95cM/d6eBXvd894oJE0w3cInq9USsyQzzoJxmiYzPAocNcuGCPGGjoXqDVJcr3C1jzt1TSjyaiLQ==", + "requires": { + "deferred-leveldown": "~5.3.0", + "level-errors": "~2.0.0", + "level-iterator-stream": "~4.0.0", + "level-supports": "~1.0.0", + "xtend": "~4.0.0" + } + }, + "leven": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/leven/-/leven-3.1.0.tgz", + "integrity": "sha512-qsda+H8jTaUaN/x5vzW2rzc+8Rw4TAQ/4KjB46IwK5VH+IlVeeeje/EoZRpiXvIqjFgK84QffqPztGI3VBLG1A==", + "dev": true + }, + "levn": { + "version": "0.4.1", "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", "integrity": "sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==", "dev": true, @@ -10597,10 +11054,16 @@ "version": "6.0.0", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "dev": true, "requires": { "yallist": "^4.0.0" } }, + "ltgt": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/ltgt/-/ltgt-2.2.1.tgz", + "integrity": "sha1-81ypHEk/e3PaDgdJUwTxezH4fuU=" + }, "lunr": { "version": "2.3.9", "resolved": "https://registry.npmjs.org/lunr/-/lunr-2.3.9.tgz", @@ -10937,14 +11400,12 @@ "mime-db": { "version": "1.46.0", "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.46.0.tgz", - "integrity": "sha512-svXaP8UQRZ5K7or+ZmfNhg2xX3yKDMUzqadsSqi4NCH/KomcH75MAMYAGVlvXn4+b/xOPhS3I2uHKRUzvjY7BQ==", - "dev": true + "integrity": "sha512-svXaP8UQRZ5K7or+ZmfNhg2xX3yKDMUzqadsSqi4NCH/KomcH75MAMYAGVlvXn4+b/xOPhS3I2uHKRUzvjY7BQ==" }, "mime-types": { "version": "2.1.29", "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.29.tgz", "integrity": "sha512-Y/jMt/S5sR9OaqteJtslsFZKWOIIqMACsJSiHghlCAyhf7jfVYjKBmLiX8OgpWeW+fjJ2b+Az69aPFPkUOY6xQ==", - "dev": true, "requires": { "mime-db": "1.46.0" } @@ -10974,7 +11435,6 @@ "version": "3.0.4", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", - "dev": true, "requires": { "brace-expansion": "^1.1.7" } @@ -10982,8 +11442,7 @@ "minimist": { "version": "1.2.5", "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", - "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==", - "dev": true + "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==" }, "minimist-options": { "version": "4.1.0", @@ -11306,6 +11765,22 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, + "multileveldown": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/multileveldown/-/multileveldown-3.0.0.tgz", + "integrity": "sha512-F1R1jotuSM0kD3G5HOGTzJtjv3oHLLkKX6Gy+3fjKxah0Ami1WBR2ZwURbXwo/01XXDfgyBP0ankc7Yn+boSwA==", + "requires": { + "abstract-leveldown": "^6.2.2", + "duplexify": "^4.1.1", + "encoding-down": "^6.3.0", + "end-of-stream": "^1.1.0", + "length-prefixed-stream": "^2.0.0", + "levelup": "^4.3.2", + "numeric-id-map": "^1.1.0", + "protocol-buffers-encodings": "^1.1.0", + "reachdown": "^1.0.0" + } + }, "mute-stream": { "version": "0.0.8", "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.8.tgz", @@ -11336,12 +11811,37 @@ "to-regex": "^3.0.1" } }, + "napi-macros": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/napi-macros/-/napi-macros-2.0.0.tgz", + "integrity": "sha512-A0xLykHtARfueITVDernsAWdtIMbOJgKgcluwENp3AlsKN/PloyO10HtmoqnFAQAcxPkgZN7wdfPfEd0zNGxbg==" + }, "natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", "integrity": "sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc=", "dev": true }, + "needle": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/needle/-/needle-2.6.0.tgz", + "integrity": "sha512-KKYdza4heMsEfSWD7VPUIz3zX2XDwOyX2d+geb4vrERZMT5RMU6ujjaD+I5Yr54uZxQ2w6XRTAhHBbSCyovZBg==", + "requires": { + "debug": "^3.2.6", + "iconv-lite": "^0.4.4", + "sax": "^1.2.4" + }, + "dependencies": { + "debug": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", + "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", + "requires": { + "ms": "^2.1.1" + } + } + } + }, "negotiator": { "version": "0.6.2", "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.2.tgz", @@ -11465,6 +11965,81 @@ "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz", "integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==" }, + "node-gyp": { + "version": "3.8.0", + "resolved": "https://registry.npmjs.org/node-gyp/-/node-gyp-3.8.0.tgz", + "integrity": "sha512-3g8lYefrRRzvGeSowdJKAKyks8oUpLEd/DyPV4eMhVlhJ0aNaZqIrNUIPuEWWTAoPqyFkfGrM67MC69baqn6vA==", + "optional": true, + "requires": { + "fstream": "^1.0.0", + "glob": "^7.0.3", + "graceful-fs": "^4.1.2", + "mkdirp": "^0.5.0", + "nopt": "2 || 3", + "npmlog": "0 || 1 || 2 || 3 || 4", + "osenv": "0", + "request": "^2.87.0", + "rimraf": "2", + "semver": "~5.3.0", + "tar": "^2.0.0", + "which": "1" + }, + "dependencies": { + "mkdirp": { + "version": "0.5.5", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz", + "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==", + "optional": true, + "requires": { + "minimist": "^1.2.5" + } + }, + "nopt": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/nopt/-/nopt-3.0.6.tgz", + "integrity": "sha1-xkZdvwirzU2zWTF/eaxopkayj/k=", + "optional": true, + "requires": { + "abbrev": "1" + } + }, + "rimraf": { + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz", + "integrity": "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==", + "optional": true, + "requires": { + "glob": "^7.1.3" + } + }, + "semver": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.3.0.tgz", + "integrity": "sha1-myzl094C0XxgEq0yaqa00M9U+U8=", + "optional": true + }, + "tar": { + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/tar/-/tar-2.2.2.tgz", + "integrity": "sha512-FCEhQ/4rE1zYv9rYXJw/msRqsnmlje5jHP6huWeBZ704jUTy02c5AZyWujpMR1ax6mVw9NyJMfuK2CMDWVIfgA==", + "optional": true, + "requires": { + "block-stream": "*", + "fstream": "^1.0.12", + "inherits": "2" + } + }, + "which": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz", + "integrity": "sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==", + "optional": true, + "requires": { + "isexe": "^2.0.0" + } + } + } + }, "node-gyp-build": { "version": "4.2.3", "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.2.3.tgz", @@ -11509,6 +12084,90 @@ } } }, + "node-pre-gyp": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/node-pre-gyp/-/node-pre-gyp-0.11.0.tgz", + "integrity": "sha512-TwWAOZb0j7e9eGaf9esRx3ZcLaE5tQ2lvYy1pb5IAaG1a2e2Kv5Lms1Y4hpj+ciXJRofIxxlt5haeQ/2ANeE0Q==", + "requires": { + "detect-libc": "^1.0.2", + "mkdirp": "^0.5.1", + "needle": "^2.2.1", + "nopt": "^4.0.1", + "npm-packlist": "^1.1.6", + "npmlog": "^4.0.2", + "rc": "^1.2.7", + "rimraf": "^2.6.1", + "semver": "^5.3.0", + "tar": "^4" + }, + "dependencies": { + "chownr": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", + "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==" + }, + "fs-minipass": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-1.2.7.tgz", + "integrity": "sha512-GWSSJGFy4e9GUeCcbIkED+bgAoFyj7XF1mV8rma3QW4NIqX9Kyx79N/PF61H5udOV3aY1IaMLs6pGbH71nlCTA==", + "requires": { + "minipass": "^2.6.0" + } + }, + "minipass": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-2.9.0.tgz", + "integrity": "sha512-wxfUjg9WebH+CUDX/CdbRlh5SmfZiy/hpkxaRI16Y9W56Pa75sWgd/rvFilSgrauD9NyFymP/+JFV3KwzIsJeg==", + "requires": { + "safe-buffer": "^5.1.2", + "yallist": "^3.0.0" + } + }, + "minizlib": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-1.3.3.tgz", + "integrity": "sha512-6ZYMOEnmVsdCeTJVE0W9ZD+pVnE8h9Hma/iOwwRDsdQoePpoX56/8B6z3P9VNwppJuBKNRuFDRNRqRWexT9G9Q==", + "requires": { + "minipass": "^2.9.0" + } + }, + "mkdirp": { + "version": "0.5.5", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz", + "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==", + "requires": { + "minimist": "^1.2.5" + } + }, + "rimraf": { + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz", + "integrity": "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==", + "requires": { + "glob": "^7.1.3" + } + }, + "tar": { + "version": "4.4.13", + "resolved": "https://registry.npmjs.org/tar/-/tar-4.4.13.tgz", + "integrity": "sha512-w2VwSrBoHa5BsSyH+KxEqeQBAllHhccyMFVHtGtdMpF4W7IRWfZjFiQceJPChOeTsSDVUpER2T8FA93pr0L+QA==", + "requires": { + "chownr": "^1.1.1", + "fs-minipass": "^1.2.5", + "minipass": "^2.8.6", + "minizlib": "^1.2.1", + "mkdirp": "^0.5.0", + "safe-buffer": "^5.1.2", + "yallist": "^3.0.3" + } + }, + "yallist": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", + "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==" + } + } + }, "node-releases": { "version": "1.1.71", "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-1.1.71.tgz", @@ -11533,6 +12192,15 @@ "webcrypto-core": "^1.1.6" } }, + "nopt": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/nopt/-/nopt-4.0.3.tgz", + "integrity": "sha512-CvaGwVMztSMJLOeXPrez7fyfObdZqNUK1cPAEzLHrTybIua9pMdmmPR5YwtfNftIOMv3DPUhFaxsZMNTQO20Kg==", + "requires": { + "abbrev": "1", + "osenv": "^0.1.4" + } + }, "normalize-package-data": { "version": "2.5.0", "resolved": "https://registry.npmjs.org/normalize-package-data/-/normalize-package-data-2.5.0.tgz", @@ -11559,6 +12227,29 @@ "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", "dev": true }, + "npm-bundled": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/npm-bundled/-/npm-bundled-1.1.2.tgz", + "integrity": "sha512-x5DHup0SuyQcmL3s7Rx/YQ8sbw/Hzg0rj48eN0dV7hf5cmQq5PXIeioroH3raV1QC1yh3uTYuMThvEQF3iKgGQ==", + "requires": { + "npm-normalize-package-bin": "^1.0.1" + } + }, + "npm-normalize-package-bin": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/npm-normalize-package-bin/-/npm-normalize-package-bin-1.0.1.tgz", + "integrity": "sha512-EPfafl6JL5/rU+ot6P3gRSCpPDW5VmIzX959Ob1+ySFUuuYHWHekXpwdUZcKP5C+DS4GEtdJluwBjnsNDl+fSA==" + }, + "npm-packlist": { + "version": "1.4.8", + "resolved": "https://registry.npmjs.org/npm-packlist/-/npm-packlist-1.4.8.tgz", + "integrity": "sha512-5+AZgwru5IevF5ZdnFglB5wNlHG1AOOuw28WhUq8/8emhBmLv6jX5by4WJCh7lW0uSYZYS6DXqIsyZVIXRZU9A==", + "requires": { + "ignore-walk": "^3.0.1", + "npm-bundled": "^1.0.1", + "npm-normalize-package-bin": "^1.0.1" + } + }, "npm-run-path": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-2.0.2.tgz", @@ -11576,6 +12267,27 @@ } } }, + "npmlog": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/npmlog/-/npmlog-4.1.2.tgz", + "integrity": "sha512-2uUqazuKlTaSI/dC8AzicUck7+IrEaOnN/e0jd3Xtt1KcGpwx30v50mL7oPyr/h9bL3E4aZccVwpwP+5W9Vjkg==", + "requires": { + "are-we-there-yet": "~1.1.2", + "console-control-strings": "~1.1.0", + "gauge": "~2.7.3", + "set-blocking": "~2.0.0" + } + }, + "number-is-nan": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz", + "integrity": "sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0=" + }, + "numeric-id-map": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/numeric-id-map/-/numeric-id-map-1.1.0.tgz", + "integrity": "sha1-ERDkRFrmg+g9R56t0PbZJJ/aO9Y=" + }, "nwsapi": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/nwsapi/-/nwsapi-2.2.0.tgz", @@ -11585,14 +12297,12 @@ "oauth-sign": { "version": "0.9.0", "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz", - "integrity": "sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==", - "dev": true + "integrity": "sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==" }, "object-assign": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", - "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=", - "dev": true + "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=" }, "object-copy": { "version": "0.1.0", @@ -11728,6 +12438,7 @@ "version": "5.1.2", "resolved": "https://registry.npmjs.org/onetime/-/onetime-5.1.2.tgz", "integrity": "sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==", + "dev": true, "requires": { "mimic-fn": "^2.1.0" }, @@ -11735,7 +12446,8 @@ "mimic-fn": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", - "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==" + "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==", + "dev": true } } }, @@ -11922,6 +12634,25 @@ } } }, + "os-homedir": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/os-homedir/-/os-homedir-1.0.2.tgz", + "integrity": "sha1-/7xJiDNuDoM94MFox+8VISGqf7M=" + }, + "os-tmpdir": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz", + "integrity": "sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ=" + }, + "osenv": { + "version": "0.1.5", + "resolved": "https://registry.npmjs.org/osenv/-/osenv-0.1.5.tgz", + "integrity": "sha512-0CWcCECdMVc2Rw3U5w9ZjqX6ga6ubk1xDVKxtBQPK7wis/0F2r9T6k4ydGYhecl7YUBxBVxhL5oisPsNxAPe2g==", + "requires": { + "os-homedir": "^1.0.0", + "os-tmpdir": "^1.0.0" + } + }, "p-defer": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/p-defer/-/p-defer-1.0.0.tgz", @@ -12015,7 +12746,8 @@ "p-try": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", - "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==" + "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", + "dev": true }, "pac-proxy-agent": { "version": "4.1.0", @@ -12110,8 +12842,7 @@ "path-is-absolute": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", - "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=", - "dev": true + "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=" }, "path-key": { "version": "3.1.1", @@ -12159,8 +12890,7 @@ "performance-now": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", - "integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=", - "dev": true + "integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=" }, "picomatch": { "version": "2.2.2", @@ -12207,54 +12937,6 @@ "find-up": "^4.0.0" } }, - "pkg-up": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/pkg-up/-/pkg-up-3.1.0.tgz", - "integrity": "sha512-nDywThFk1i4BQK4twPQ6TA4RT8bDY96yeuCVBWL3ePARCiEKDRSrNGbFIgUJpLp+XeIR65v8ra7WuJOFUBtkMA==", - "requires": { - "find-up": "^3.0.0" - }, - "dependencies": { - "find-up": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/find-up/-/find-up-3.0.0.tgz", - "integrity": "sha512-1yD6RmLI1XBfxugvORwlck6f75tYL+iR0jqwsOrOxMZyGYqUuDhJ0l4AXdO1iX/FTs9cBAMEk1gWSEx1kSbylg==", - "requires": { - "locate-path": "^3.0.0" - } - }, - "locate-path": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-3.0.0.tgz", - "integrity": "sha512-7AO748wWnIhNqAuaty2ZWHkQHRSNfPVIsPIfwEOWO22AmaoVrWavlOcMR5nzTLNYvp36X220/maaRsrec1G65A==", - "requires": { - "p-locate": "^3.0.0", - "path-exists": "^3.0.0" - } - }, - "p-limit": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", - "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", - "requires": { - "p-try": "^2.0.0" - } - }, - "p-locate": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-3.0.0.tgz", - "integrity": "sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ==", - "requires": { - "p-limit": "^2.0.0" - } - }, - "path-exists": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz", - "integrity": "sha1-zg6+ql94yxiSXqfYENe1mwEP1RU=" - } - } - }, "platform": { "version": "1.3.6", "resolved": "https://registry.npmjs.org/platform/-/platform-1.3.6.tgz", @@ -12332,8 +13014,7 @@ "process-nextick-args": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", - "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", - "dev": true + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" }, "progress": { "version": "2.0.3", @@ -12356,6 +13037,22 @@ "sisteransi": "^1.0.5" } }, + "protocol-buffers-encodings": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/protocol-buffers-encodings/-/protocol-buffers-encodings-1.1.1.tgz", + "integrity": "sha512-5aFshI9SbhtcMiDiZZu3g2tMlZeS5lhni//AGJ7V34PQLU5JA91Cva7TIs6inZhYikS3OpnUzAUuL6YtS0CyDA==", + "requires": { + "signed-varint": "^2.0.1", + "varint": "5.0.0" + }, + "dependencies": { + "varint": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/varint/-/varint-5.0.0.tgz", + "integrity": "sha1-2Ca4n3SQcy+rwMDtaT7Uddyynr8=" + } + } + }, "proxy-addr": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.6.tgz", @@ -12408,14 +13105,12 @@ "prr": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/prr/-/prr-1.0.1.tgz", - "integrity": "sha1-0/wRS6BplaRexok/SEzrHXj19HY=", - "dev": true + "integrity": "sha1-0/wRS6BplaRexok/SEzrHXj19HY=" }, "psl": { "version": "1.8.0", "resolved": "https://registry.npmjs.org/psl/-/psl-1.8.0.tgz", - "integrity": "sha512-RIdOzyoavK+hA18OGGWDqUTsCLhtA7IcZ/6NCs4fFJaHBDab+pDDmDIByWFRQJq2Cd7r1OoQxBGKOaztq+hjIQ==", - "dev": true + "integrity": "sha512-RIdOzyoavK+hA18OGGWDqUTsCLhtA7IcZ/6NCs4fFJaHBDab+pDDmDIByWFRQJq2Cd7r1OoQxBGKOaztq+hjIQ==" }, "public-encrypt": { "version": "4.0.3", @@ -12516,6 +13211,29 @@ "unpipe": "1.0.0" } }, + "rc": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", + "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", + "requires": { + "deep-extend": "^0.6.0", + "ini": "~1.3.0", + "minimist": "^1.2.0", + "strip-json-comments": "~2.0.1" + }, + "dependencies": { + "strip-json-comments": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", + "integrity": "sha1-PFMZQukIwml8DsNEhYwobHygpgo=" + } + } + }, + "reachdown": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/reachdown/-/reachdown-1.1.0.tgz", + "integrity": "sha512-6LsdRe4cZyOjw4NnvbhUd/rGG7WQ9HMopPr+kyL018Uci4kijtxcGR5kVb5Ln13k4PEE+fEFQbjfOvNw7cnXmA==" + }, "react-is": { "version": "17.0.1", "resolved": "https://registry.npmjs.org/react-is/-/react-is-17.0.1.tgz", @@ -12833,7 +13551,6 @@ "version": "2.88.2", "resolved": "https://registry.npmjs.org/request/-/request-2.88.2.tgz", "integrity": "sha512-MsvtOrfG9ZcrOwAW+Qi+F6HbD0CWXEh9ou77uOb7FM2WPhwT7smM833PzanhJLsgXjN89Ir6V2PczXNnMpwKhw==", - "dev": true, "requires": { "aws-sign2": "~0.7.0", "aws4": "^1.8.0", @@ -12860,14 +13577,12 @@ "qs": { "version": "6.5.2", "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz", - "integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==", - "dev": true + "integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==" }, "tough-cookie": { "version": "2.5.0", "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.5.0.tgz", "integrity": "sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g==", - "dev": true, "requires": { "psl": "^1.1.28", "punycode": "^2.1.1" @@ -12876,8 +13591,7 @@ "uuid": { "version": "3.4.0", "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", - "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==", - "dev": true + "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==" } } }, @@ -12946,7 +13660,8 @@ "require-from-string": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", - "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==" + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "dev": true }, "require-main-filename": { "version": "2.0.0", @@ -13081,8 +13796,7 @@ "safer-buffer": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", - "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", - "dev": true + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, "sane": { "version": "4.1.0", @@ -13101,6 +13815,11 @@ "walker": "~1.0.5" } }, + "sax": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz", + "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==" + }, "saxes": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/saxes/-/saxes-5.0.1.tgz", @@ -13139,8 +13858,7 @@ "semver": { "version": "5.7.1", "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", - "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", - "dev": true + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==" }, "send": { "version": "0.17.1", @@ -13212,8 +13930,7 @@ "set-blocking": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", - "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc=", - "dev": true + "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc=" }, "set-value": { "version": "2.0.1", @@ -13327,8 +14044,15 @@ "signal-exit": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.3.tgz", - "integrity": "sha512-VUJ49FC8U1OxwZLxIbTTrDvLnf/6TDgxZcK8wxR8zs13xpx7xbG60ndBlhNrFi2EMuFRoeDoJO7wthSLq42EjA==", - "dev": true + "integrity": "sha512-VUJ49FC8U1OxwZLxIbTTrDvLnf/6TDgxZcK8wxR8zs13xpx7xbG60ndBlhNrFi2EMuFRoeDoJO7wthSLq42EjA==" + }, + "signed-varint": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/signed-varint/-/signed-varint-2.0.1.tgz", + "integrity": "sha1-UKmYnafJjCxh2tEZvJdHDvhSgSk=", + "requires": { + "varint": "~5.0.0" + } }, "sinon": { "version": "9.2.4", @@ -13689,11 +14413,32 @@ "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=", "dev": true }, + "sqlite": { + "version": "4.0.21", + "resolved": "https://registry.npmjs.org/sqlite/-/sqlite-4.0.21.tgz", + "integrity": "sha512-HIqObuvz+Vx8BXWzIhR12fJMjiE37Mdfupg2Ok0f8MChSqALXj7FgpZauj1pJoSY6qsDYmp/+/ZgSn3l8yutoA==" + }, + "sqlite3": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/sqlite3/-/sqlite3-5.0.2.tgz", + "integrity": "sha512-1SdTNo+BVU211Xj1csWa8lV6KM0CtucDwRyA0VHl91wEH1Mgh7RxUpI4rVvG7OhHrzCSGaVyW5g8vKvlrk9DJA==", + "requires": { + "node-addon-api": "^3.0.0", + "node-gyp": "3.x", + "node-pre-gyp": "^0.11.0" + }, + "dependencies": { + "node-addon-api": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-3.1.0.tgz", + "integrity": "sha512-flmrDNB06LIl5lywUz7YlNGZH/5p0M7W28k8hzd9Lshtdh1wshD2Y+U4h9LD6KObOy1f+fEVdgprPrEymjM5uw==" + } + } + }, "sshpk": { "version": "1.16.1", "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.16.1.tgz", "integrity": "sha512-HXXqVUq7+pcKeLqqZj6mHFUMvXtOJt1uoUx09pFW6011inTMxqI8BA8PM95myrIyyKwdnzjdFjLiE6KBPVtJIg==", - "dev": true, "requires": { "asn1": "~0.2.3", "assert-plus": "^1.0.0", @@ -13756,6 +14501,11 @@ "integrity": "sha1-NbCYdbT/SfJqd35QmzCQoyJr8ks=", "dev": true }, + "stream-shift": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.1.tgz", + "integrity": "sha512-AiisoFqQ0vbGcZgQPY1cdP2I76glaVA/RauYR4G4thNFgkTqr90yXTo4LYX60Jl+sIlPNHHdGSwo01AvbKUSVQ==" + }, "streamr-client-protocol": { "version": "8.0.2", "resolved": "https://registry.npmjs.org/streamr-client-protocol/-/streamr-client-protocol-8.0.2.tgz", @@ -13858,7 +14608,6 @@ "version": "3.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", - "dev": true, "requires": { "ansi-regex": "^2.0.0" }, @@ -13866,8 +14615,7 @@ "ansi-regex": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", - "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=", - "dev": true + "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=" } } }, @@ -14477,7 +15225,6 @@ "version": "0.6.0", "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", "integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=", - "dev": true, "requires": { "safe-buffer": "^5.0.1" } @@ -14485,8 +15232,7 @@ "tweetnacl": { "version": "0.14.5", "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", - "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=", - "dev": true + "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=" }, "type-check": { "version": "0.4.0", @@ -14804,6 +15550,11 @@ "spdx-expression-parse": "^3.0.0" } }, + "varint": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/varint/-/varint-5.0.2.tgz", + "integrity": "sha512-lKxKYG6H03yCZUpAGOPOsMcGxd1RHCu1iKvEHYDPmTyq2HueGhD73ssNBqqQWfvYs04G9iUFRvmAVLW20Jw6ow==" + }, "vary": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", @@ -14814,7 +15565,6 @@ "version": "1.10.0", "resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz", "integrity": "sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA=", - "dev": true, "requires": { "assert-plus": "^1.0.0", "core-util-is": "1.0.2", @@ -15225,7 +15975,6 @@ "version": "1.1.3", "resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.3.tgz", "integrity": "sha512-QGkOQc8XL6Bt5PwnsExKBPuMKBxnGxWWW3fU55Xt4feHozMUhdUMaBCk290qpm/wG5u/RSKzwdAC4i51YigihA==", - "dev": true, "requires": { "string-width": "^1.0.2 || 2" }, @@ -15233,20 +15982,17 @@ "ansi-regex": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.0.tgz", - "integrity": "sha1-7QMXwyIGT3lGbAKWa922Bas32Zg=", - "dev": true + "integrity": "sha1-7QMXwyIGT3lGbAKWa922Bas32Zg=" }, "is-fullwidth-code-point": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", - "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=", - "dev": true + "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=" }, "string-width": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz", "integrity": "sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw==", - "dev": true, "requires": { "is-fullwidth-code-point": "^2.0.0", "strip-ansi": "^4.0.0" @@ -15256,7 +16002,6 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz", "integrity": "sha1-qEeQIusaw2iocTibY1JixQXuNo8=", - "dev": true, "requires": { "ansi-regex": "^3.0.0" } @@ -15367,6 +16112,11 @@ "integrity": "sha1-UqY+VsoLhKfzpfPWGHLxJq16WUM=", "dev": true }, + "xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==" + }, "y18n": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.1.tgz", @@ -15376,7 +16126,8 @@ "yallist": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", - "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "dev": true }, "yaml": { "version": "1.10.2", diff --git a/package.json b/package.json index 5daa95179..5740e2aaf 100644 --- a/package.json +++ b/package.json @@ -150,9 +150,12 @@ "@ethersproject/transactions": "^5.1.1", "@ethersproject/wallet": "^5.1.0", "@ethersproject/web": "^5.1.0", - "conf": "^9.0.2", + "@types/level": "^6.0.0", "debug": "^4.3.2", + "env-paths": "^2.2.1", "eventemitter3": "^4.0.7", + "level": "^7.0.0", + "level-party": "^5.0.0", "lodash": "^4.17.21", "mem": "^8.1.1", "node-abort-controller": "^1.2.1", @@ -166,6 +169,8 @@ "qs": "^6.10.1", "quick-lru": "^6.0.0", "readable-stream": "^3.6.0", + "sqlite": "^4.0.21", + "sqlite3": "^5.0.2", "streamr-client-protocol": "^8.0.2", "streamr-test-utils": "^1.3.2", "ts-toolbelt": "^9.6.0", diff --git a/src/publish/Encrypt.ts b/src/publish/Encrypt.ts index 26ff27482..f9cb8fca1 100644 --- a/src/publish/Encrypt.ts +++ b/src/publish/Encrypt.ts @@ -28,9 +28,19 @@ export default function Encrypt(client: StreamrClient) { return } + const { messageType } = streamMessage + if ( + messageType === StreamMessage.MESSAGE_TYPES.GROUP_KEY_RESPONSE + || messageType === StreamMessage.MESSAGE_TYPES.GROUP_KEY_REQUEST + || messageType === StreamMessage.MESSAGE_TYPES.GROUP_KEY_ERROR_RESPONSE + ) { + // never encrypt + return + } + if ( !stream.requireEncryptedData - && !getPublisherKeyExchange().hasAnyGroupKey(stream.id) + && !(await (getPublisherKeyExchange().hasAnyGroupKey(stream.id))) ) { // not needed return diff --git a/src/stream/KeyExchange.ts b/src/stream/KeyExchange.ts index 32cd8b39c..ee099291a 100644 --- a/src/stream/KeyExchange.ts +++ b/src/stream/KeyExchange.ts @@ -2,6 +2,7 @@ import { StreamMessage, GroupKeyRequest, GroupKeyResponse, GroupKeyErrorResponse, EncryptedGroupKey, Errors } from 'streamr-client-protocol' import mem from 'mem' +import pMemoize from 'p-memoize' import { uuid, Defer } from '../utils' import Scaffold from '../utils/Scaffold' @@ -10,6 +11,7 @@ import { validateOptions } from './utils' import EncryptionUtil, { GroupKey, GroupKeyish, StreamMessageProcessingError } from './Encryption' import type { Subscription } from '../subscribe' import { StreamrClient } from '../StreamrClient' +import PersistentStore from './PersistentStore' const KEY_EXCHANGE_STREAM_PREFIX = 'SYSTEM/keyexchange' @@ -63,25 +65,29 @@ function getKeyExchangeStreamId(address: Address) { } type GroupKeyStoreOptions = { - id: string, + clientId: string, + streamId: string, groupKeys: [GroupKeyId, GroupKey][] } -function GroupKeyStore({ groupKeys }: GroupKeyStoreOptions) { - const store = new Map(groupKeys) +function GroupKeyStore({ clientId, streamId, groupKeys }: GroupKeyStoreOptions) { + const store = new PersistentStore({ clientId, streamId }) let currentGroupKeyId: GroupKeyId | undefined // current key id if any const nextGroupKeys: GroupKey[] = [] // the keys to use next, disappears if not actually used. Max queue size 2 - store.forEach((groupKey) => { + groupKeys.forEach(([groupKeyId, groupKey]) => { GroupKey.validate(groupKey) + if (groupKeyId !== groupKey.id) { + throw new Error(`Ids must match: groupKey.id: ${groupKey.id}, groupKeyId: ${groupKeyId}`) + } // use last init key as current currentGroupKeyId = groupKey.id }) - function storeKey(groupKey: GroupKey) { + async function storeKey(groupKey: GroupKey) { GroupKey.validate(groupKey) - const existingKey = store.get(groupKey.id) + const existingKey = await store.get(groupKey.id) if (existingKey) { if (!existingKey.equals(groupKey)) { throw new GroupKey.InvalidGroupKeyError( @@ -90,49 +96,48 @@ function GroupKeyStore({ groupKeys }: GroupKeyStoreOptions) { ) } - store.delete(groupKey.id) // sort key at end by deleting existing entry before re-adding - store.set(groupKey.id, existingKey) // reuse existing instance + await store.set(groupKey.id, existingKey) return existingKey } - store.set(groupKey.id, groupKey) + await store.set(groupKey.id, groupKey) return groupKey } return { - has(id: GroupKeyId) { + async has(id: GroupKeyId) { if (currentGroupKeyId === id) { return true } if (nextGroupKeys.some((nextKey) => nextKey.id === id)) { return true } return store.has(id) }, - isEmpty() { - return nextGroupKeys.length === 0 && store.size === 0 + async isEmpty() { + return !nextGroupKeys.length && await store.size() === 0 }, - useGroupKey(): Promise<[GroupKey, GroupKey | undefined]> { + async useGroupKey(): Promise<[GroupKey, GroupKey | undefined]> { const nextGroupKey = nextGroupKeys.pop() switch (true) { // First use of group key on this stream, no current key. Make next key current. case !!(!currentGroupKeyId && nextGroupKey): { - storeKey(nextGroupKey) + await storeKey(nextGroupKey) currentGroupKeyId = nextGroupKey.id return [ - this.get(currentGroupKeyId), + await this.get(currentGroupKeyId), undefined, ] } // Keep using current key (empty next) case !!(currentGroupKeyId && !nextGroupKey): { return [ - this.get(currentGroupKeyId), + await this.get(currentGroupKeyId), undefined ] } // Key changed (non-empty next). return current + next. Make next key current. case !!(currentGroupKeyId && nextGroupKey): { - storeKey(nextGroupKey) - const prevGroupKey = this.get(currentGroupKeyId) + await storeKey(nextGroupKey) + const prevGroupKey = await this.get(currentGroupKeyId) currentGroupKeyId = nextGroupKey.id // use current key one more time return [ @@ -142,33 +147,33 @@ function GroupKeyStore({ groupKeys }: GroupKeyStoreOptions) { } // Generate & use new key if none already set. default: { - this.rotateGroupKey() + await this.rotateGroupKey() return this.useGroupKey() } } }, - get(id: GroupKeyId) { + async get(id: GroupKeyId) { return store.get(id) }, - clear() { + async clear() { currentGroupKeyId = undefined nextGroupKeys.length = 0 return store.clear() }, - rotateGroupKey() { + async rotateGroupKey() { return this.setNextGroupKey(GroupKey.generate()) }, - add(groupKey: GroupKey) { + async add(groupKey: GroupKey) { return storeKey(groupKey) }, - setNextGroupKey(newKey: GroupKey) { + async setNextGroupKey(newKey: GroupKey) { GroupKey.validate(newKey) nextGroupKeys.unshift(newKey) nextGroupKeys.length = Math.min(nextGroupKeys.length, 2) }, - rekey() { + async rekey() { const newKey = GroupKey.generate() - storeKey(newKey) + await storeKey(newKey) currentGroupKeyId = newKey.id nextGroupKeys.length = 0 } @@ -239,7 +244,7 @@ async function catchKeyExchangeError(client: StreamrClient, streamMessage: Strea } } -async function PublisherKeyExhangeSubscription(client: StreamrClient, getGroupKeyStore: (streamId: string) => GroupKeyStorage) { +async function PublisherKeyExhangeSubscription(client: StreamrClient, getGroupKeyStore: (streamId: string) => Promise) { async function onKeyExchangeMessage(_parsedContent: any, streamMessage: StreamMessage) { return catchKeyExchangeError(client, streamMessage, async () => { if (streamMessage.messageType !== StreamMessage.MESSAGE_TYPES.GROUP_KEY_REQUEST) { @@ -251,16 +256,16 @@ async function PublisherKeyExhangeSubscription(client: StreamrClient, getGroupKe const subscriberId = streamMessage.getPublisherId() - const groupKeyStore = getGroupKeyStore(streamId) - const isSubscriber = await client.isStreamSubscriber(streamId, subscriberId) - const encryptedGroupKeys = !isSubscriber ? [] : groupKeyIds.map((id) => { - const groupKey = groupKeyStore.get(id) + const groupKeyStore = await getGroupKeyStore(streamId) + const isSubscriber = await client.isStreamSubscriber(streamId, subscriberId) + const encryptedGroupKeys = (!isSubscriber ? [] : await Promise.all(groupKeyIds.map(async (id) => { + const groupKey = await groupKeyStore.get(id) if (!groupKey) { return null // will be filtered out } const key = EncryptionUtil.encryptWithPublicKey(groupKey.data, rsaPublicKey, true) return new EncryptedGroupKey(id, key) - }).filter(Boolean) as EncryptedGroupKey[] + }))).filter(Boolean) as EncryptedGroupKey[] client.debug('Publisher: Subscriber requested groupKeys: %d. Got: %d. %o', groupKeyIds.length, encryptedGroupKeys.length, { subscriberId, @@ -310,15 +315,20 @@ type KeyExhangeOptions = { export function PublisherKeyExhange(client: StreamrClient, { groupKeys = {} }: KeyExhangeOptions = {}) { let enabled = true - const getGroupKeyStore = mem((streamId) => GroupKeyStore({ - id: `${client.id}:${streamId}`, - groupKeys: [...parseGroupKeys(groupKeys![streamId]).entries()], - }), { + const getGroupKeyStore = pMemoize(async (streamId) => { + const clientId = await client.getAddress() + return GroupKeyStore({ + clientId, + streamId, + groupKeys: [...parseGroupKeys(groupKeys[streamId]).entries()] + }) + }, { cacheKey([maybeStreamId]) { const { streamId } = validateOptions(maybeStreamId) return streamId } }) + let sub: Subscription | undefined const next = Scaffold([ async () => { @@ -332,29 +342,28 @@ export function PublisherKeyExhange(client: StreamrClient, { groupKeys = {} }: K } ], async () => enabled) - function rotateGroupKey(streamId: string) { + async function rotateGroupKey(streamId: string) { if (!enabled) { return } - const groupKeyStore = getGroupKeyStore(streamId) - groupKeyStore.rotateGroupKey() + const groupKeyStore = await getGroupKeyStore(streamId) + await groupKeyStore.rotateGroupKey() } - function setNextGroupKey(streamId: string, groupKey: GroupKey) { + async function setNextGroupKey(streamId: string, groupKey: GroupKey) { if (!enabled) { return } - const groupKeyStore = getGroupKeyStore(streamId) + const groupKeyStore = await getGroupKeyStore(streamId) - groupKeyStore.setNextGroupKey(groupKey) + await groupKeyStore.setNextGroupKey(groupKey) } async function useGroupKey(streamId: string) { await next() if (!enabled) { return undefined } - const groupKeyStore = getGroupKeyStore(streamId) - + const groupKeyStore = await getGroupKeyStore(streamId) return groupKeyStore.useGroupKey() } - function hasAnyGroupKey(streamId: string) { - const groupKeyStore = getGroupKeyStore(streamId) + async function hasAnyGroupKey(streamId: string) { + const groupKeyStore = await getGroupKeyStore(streamId) return !groupKeyStore.isEmpty() } @@ -376,6 +385,7 @@ export function PublisherKeyExhange(client: StreamrClient, { groupKeys = {} }: K return next() }, async stop() { + pMemoize.clear(getGroupKeyStore) enabled = false return next() } @@ -385,13 +395,16 @@ export function PublisherKeyExhange(client: StreamrClient, { groupKeys = {} }: K async function getGroupKeysFromStreamMessage(streamMessage: StreamMessage, encryptionUtil: EncryptionUtil) { const { encryptedGroupKeys } = GroupKeyResponse.fromArray(streamMessage.getParsedContent()) return Promise.all(encryptedGroupKeys.map(async (encryptedGroupKey) => ( - new GroupKey(encryptedGroupKey.groupKeyId, await encryptionUtil.decryptWithPrivateKey(encryptedGroupKey.encryptedGroupKeyHex, true)) + new GroupKey( + encryptedGroupKey.groupKeyId, + await encryptionUtil.decryptWithPrivateKey(encryptedGroupKey.encryptedGroupKeyHex, true) + ) ))) } async function SubscriberKeyExhangeSubscription( client: StreamrClient, - getGroupKeyStore: (streamId: string) => GroupKeyStorage, + getGroupKeyStore: (streamId: string) => Promise, encryptionUtil: EncryptionUtil ) { let sub: Subscription @@ -404,10 +417,10 @@ async function SubscriberKeyExhangeSubscription( } const groupKeys = await getGroupKeysFromStreamMessage(streamMessage, encryptionUtil) - const groupKeyStore = getGroupKeyStore(streamMessage.getStreamId()) - groupKeys.forEach((groupKey) => { + const groupKeyStore = await getGroupKeyStore(streamMessage.getStreamId()) + await Promise.all(groupKeys.map(async (groupKey) => ( groupKeyStore.add(groupKey) - }) + ))) } catch (err) { sub.emit('error', err) } @@ -422,10 +435,14 @@ export function SubscriberKeyExchange(client: StreamrClient, { groupKeys = {} }: let enabled = true const encryptionUtil = new EncryptionUtil(client.options.keyExchange) - const getGroupKeyStore = mem((streamId) => GroupKeyStore({ - id: client.id, - groupKeys: [...parseGroupKeys(groupKeys[streamId]).entries()] - }), { + const getGroupKeyStore = pMemoize(async (streamId) => { + const clientId = await client.getAddress() + return GroupKeyStore({ + clientId, + streamId, + groupKeys: [...parseGroupKeys(groupKeys[streamId]).entries()] + }) + }, { cacheKey([maybeStreamId]) { const { streamId } = validateOptions(maybeStreamId) return streamId @@ -518,8 +535,8 @@ export function SubscriberKeyExchange(client: StreamrClient, { groupKeys = {} }: if (!groupKeyId) { return Promise.resolve() } - const groupKeyStore = getGroupKeyStore(streamId) - if (groupKeyStore.has(groupKeyId)) { + const groupKeyStore = await getGroupKeyStore(streamId) + if (await groupKeyStore.has(groupKeyId)) { return groupKeyStore.get(groupKeyId) } @@ -542,16 +559,17 @@ export function SubscriberKeyExchange(client: StreamrClient, { groupKeys = {} }: publisherId, groupKeyIds, }) - receivedGroupKeys.forEach((groupKey) => { + await Promise.all(receivedGroupKeys.map(async (groupKey) => ( groupKeyStore.add(groupKey) - }) - groupKeyIds.forEach((id) => { + ))) + await Promise.all(groupKeyIds.map(async (id) => { if (!pending.has(id)) { return } - const groupKey = groupKeyStore.get(id) + const groupKeyTask = groupKeyStore.get(id) const task = pending.get(id) pending.delete(id) + const groupKey = await groupKeyTask task.resolve(groupKey) - }) + })) } catch (err) { groupKeyIds.forEach((id) => { if (!pending.has(id)) { return } @@ -579,7 +597,7 @@ export function SubscriberKeyExchange(client: StreamrClient, { groupKeys = {} }: async () => { sub = await SubscriberKeyExhangeSubscription(client, getGroupKeyStore, encryptionUtil) return async () => { - mem.clear(getGroupKeyStore) + pMemoize.clear(getGroupKeyStore) if (!sub) { return } const cancelTask = sub.cancel() sub = undefined diff --git a/src/stream/PersistentStore.ts b/src/stream/PersistentStore.ts index 3d47539fd..c4282c977 100644 --- a/src/stream/PersistentStore.ts +++ b/src/stream/PersistentStore.ts @@ -1,65 +1,121 @@ -import Conf from 'conf' +import { once } from 'events' +import envPaths from 'env-paths' +import Level from 'level' +// @ts-expect-error +import LevelParty from 'level-party' +import { dirname, join } from 'path' +import fs from 'fs/promises' -export interface PersistentStorage extends Map { - id: string -} +import { GroupKey } from './Encryption' +import { pOnce } from '../utils' + +class ServerStorage { + readonly id: string + readonly dbFilePath: string + private readonly store: Level.LevelDB + private error?: Error -export default class ServerStorage implements PersistentStorage { - id: string - config: Conf constructor(id: string) { - this.id = id - this.config = new Conf({ - projectName: 'streamr-client', - configName: id, + this.id = encodeURIComponent(id) + const paths = envPaths('streamr-client') + const dbFilePath = join(paths.data, `${id}.db`) + this.dbFilePath = dbFilePath + const Store = LevelParty as Level.Constructor + this.store = Store(dbFilePath, { valueEncoding: 'json' }, (err) => { + this.error = err + }) + + this.init = pOnce(this.init.bind(this)) + } + + async init() { + try { + await fs.mkdir(dirname(this.dbFilePath), { recursive: true }) + await this.store.open() + } catch (err) { + if (!this.error) { + this.error = err + } + } + + if (this.error) { + throw this.error + } + } + + async get(key: string) { + await this.init() + const value = await this.store.get(key).catch((err) => { + if (err.notFound) { return } + throw err }) + return value + } + + async set(key: string, value: any) { + await this.init() + return this.store.put(key, value) } - has(key: string) { - return this.config.has(key) + async delete(key: string) { + await this.init() + return this.store.del(key).catch((err) => { + if (err.notFound) { return } + throw err + }) } - get(key: string) { - return this.config.get(key) + async clear() { + await this.init() + return this.store.clear() } - keys() { - return Object.keys(this.config.store)[Symbol.iterator]() + async size() { + await this.init() + let count = 0 + const keyStream = this.store.createKeyStream({ keys: false, values: true }).on('data', () => { + count += 1 + }) + await once(keyStream, 'end') + return count } - values() { - return Object.values(this.config.store)[Symbol.iterator]() + get [Symbol.toStringTag]() { + return this.constructor.name } +} - entries() { - return Object.entries(this.config.store)[Symbol.iterator]() +export default class GroupKeyStore { + store: ServerStorage + constructor({ clientId, streamId }: { clientId: string, streamId: string }) { + this.store = new ServerStorage(`${clientId}-${streamId}`) } - forEach(...args: Parameters['forEach']>) { - return new Map(Object.entries(this.config.store)).forEach(...args) + async has(groupKeyId: string) { + const value = await this.store.get(groupKeyId) + return value != null } - set(key: string, value: any) { - this.config.set(key, value) - return this + async size() { + return this.store.size() } - delete(key: string) { - const had = this.config.has(key) - this.config.delete(key) - return had + async get(groupKeyId: string) { + const value = await this.store.get(groupKeyId) + if (!value) { return undefined } + return GroupKey.from(value) } - clear() { - return this.config.clear() + async set(groupKeyId: string, value: GroupKey) { + return this.store.set(groupKeyId, value) } - get size() { - return this.config.size + async delete(groupKeyId: string) { + return this.store.delete(groupKeyId) } - [Symbol.iterator]() { - return new Map(Object.entries(this.config.store))[Symbol.iterator]() + async clear() { + return this.store.clear() } get [Symbol.toStringTag]() { diff --git a/src/utils/index.ts b/src/utils/index.ts index caf748c8b..9a637eef4 100644 --- a/src/utils/index.ts +++ b/src/utils/index.ts @@ -389,6 +389,46 @@ export function pOne(fn: F.Function) { } } +type Unwrap = T extends Promise ? U : T + +/** + * Only allows calling `fn` once. + * Returns same promise while task is executing. + */ + +export function pOnce( + fn: (...args: Args) => R +): (...args: Args) => Promise> { + let inProgress: Promise | undefined + let started = false + let value: Unwrap + let error: Error | undefined + return async (...args: Args) => { + if (!started) { + started = true + inProgress = (async () => { + try { + value = await Promise.resolve(fn(...args)) as Unwrap + } catch (err) { + error = err + } finally { + inProgress = undefined + } + })() + } + + if (inProgress) { + await inProgress + } + + if (error) { + throw error + } + + return value + } +} + export class TimeoutError extends Error { timeout: number constructor(msg = '', timeout = 0) { From a7fa7a3a876c3bb1bf38525eefa0c7fffd297660 Mon Sep 17 00:00:00 2001 From: Tim Oxley Date: Tue, 4 May 2021 16:06:02 -0400 Subject: [PATCH 13/28] fix(keyexchange): Improve subscriber async cleanup. --- src/stream/KeyExchange.ts | 33 +++++++++++++++++++++++++++++---- src/subscribe/Decrypt.js | 2 +- src/subscribe/pipeline.js | 1 + 3 files changed, 31 insertions(+), 5 deletions(-) diff --git a/src/stream/KeyExchange.ts b/src/stream/KeyExchange.ts index ee099291a..1f0cf340b 100644 --- a/src/stream/KeyExchange.ts +++ b/src/stream/KeyExchange.ts @@ -536,8 +536,13 @@ export function SubscriberKeyExchange(client: StreamrClient, { groupKeys = {} }: return Promise.resolve() } const groupKeyStore = await getGroupKeyStore(streamId) - if (await groupKeyStore.has(groupKeyId)) { - return groupKeyStore.get(groupKeyId) + + if (!enabled) { return Promise.resolve() } + const existingGroupKey = await groupKeyStore.get(groupKeyId) + if (!enabled) { return Promise.resolve() } + + if (existingGroupKey) { + return existingGroupKey } if (pending.has(groupKeyId)) { @@ -550,6 +555,7 @@ export function SubscriberKeyExchange(client: StreamrClient, { groupKeys = {} }: pending.set(groupKeyId, Defer()) async function processBuffer() { + if (!enabled) { return } const currentBuffer = getBuffer(key) const groupKeyIds = currentBuffer.slice() currentBuffer.length = 0 @@ -559,9 +565,11 @@ export function SubscriberKeyExchange(client: StreamrClient, { groupKeys = {} }: publisherId, groupKeyIds, }) + if (!enabled) { return } await Promise.all(receivedGroupKeys.map(async (groupKey) => ( groupKeyStore.add(groupKey) ))) + if (!enabled) { return } await Promise.all(groupKeyIds.map(async (id) => { if (!pending.has(id)) { return } const groupKeyTask = groupKeyStore.get(id) @@ -590,14 +598,26 @@ export function SubscriberKeyExchange(client: StreamrClient, { groupKeys = {} }: return pending.get(groupKeyId) } + function cleanupPending() { + Array.from(Object.entries(timeouts)).forEach(([key, value]) => { + clearTimeout(value) + delete timeouts[key] + }) + const pendingValues = Array.from(pending.values()) + pending.clear() + pendingValues.forEach((value) => { + value.resolve(undefined) + }) + pMemoize.clear(getGroupKeyStore) + } + const next = Scaffold([ async () => { - return encryptionUtil.onReady() + await encryptionUtil.onReady() }, async () => { sub = await SubscriberKeyExhangeSubscription(client, getGroupKeyStore, encryptionUtil) return async () => { - pMemoize.clear(getGroupKeyStore) if (!sub) { return } const cancelTask = sub.cancel() sub = undefined @@ -606,6 +626,11 @@ export function SubscriberKeyExchange(client: StreamrClient, { groupKeys = {} }: } ], async () => enabled, { id: `SubscriberKeyExhangeSubscription.${client.id}`, + onChange(shouldUp) { + if (!shouldUp) { + cleanupPending() + } + }, async onDone() { // clean up requestKey if (requestKeysStep) { diff --git a/src/subscribe/Decrypt.js b/src/subscribe/Decrypt.js index 6e97f9f76..a670f4201 100644 --- a/src/subscribe/Decrypt.js +++ b/src/subscribe/Decrypt.js @@ -56,7 +56,7 @@ export default function Decrypt(client, options = {}) { } return Object.assign(decrypt, { - stop() { + async stop() { return requestKey.stop() } }) diff --git a/src/subscribe/pipeline.js b/src/subscribe/pipeline.js index 745d2e02f..71744adc6 100644 --- a/src/subscribe/pipeline.js +++ b/src/subscribe/pipeline.js @@ -129,6 +129,7 @@ export default function MessagePipeline(client, opts = {}, onFinally = async (er // custom pipeline steps ...afterSteps ], async (err, ...args) => { + decrypt.stop() await msgStream.cancel(err) try { if (err) { From 6ef29fb88350b7af52d24bc320014459231d3d53 Mon Sep 17 00:00:00 2001 From: Tim Oxley Date: Tue, 4 May 2021 16:06:27 -0400 Subject: [PATCH 14/28] test(keyexchange): Add EncryptionKeyPersistence.test.ts --- .../EncryptionKeyPersistence.test.ts | 280 ++++++++++++++++++ 1 file changed, 280 insertions(+) create mode 100644 test/integration/EncryptionKeyPersistence.test.ts diff --git a/test/integration/EncryptionKeyPersistence.test.ts b/test/integration/EncryptionKeyPersistence.test.ts new file mode 100644 index 000000000..0596b34eb --- /dev/null +++ b/test/integration/EncryptionKeyPersistence.test.ts @@ -0,0 +1,280 @@ +import { wait } from 'streamr-test-utils' + +import { describeRepeats, fakePrivateKey, uid, getPublishTestMessages, addAfterFn } from '../utils' +// import { Defer } from '../../src/utils' +import { StreamrClient } from '../../src/StreamrClient' +import { GroupKey } from '../../src/stream/Encryption' +import Connection from '../../src/Connection' + +import config from './config' + +const TIMEOUT = 30 * 1000 + +describeRepeats('decryption', () => { + let publishTestMessages + let expectErrors = 0 // check no errors by default + let errors = [] + let subscriber + const addAfter = addAfterFn() + + const getOnError = (errs) => jest.fn((err) => { + errs.push(err) + }) + + let onError = jest.fn() + let publisher + let stream + + const createClient = (opts = {}) => { + const c = new StreamrClient({ + ...config.clientOptions, + auth: { + privateKey: fakePrivateKey(), + }, + autoConnect: false, + autoDisconnect: false, + // @ts-expect-error + disconnectDelay: 1, + publishAutoDisconnectDelay: 50, + maxRetries: 2, + ...opts, + }) + c.onError = jest.fn() + c.on('error', onError) + + return c + } + + beforeEach(() => { + errors = [] + expectErrors = 0 + onError = getOnError(errors) + }) + + afterEach(async () => { + await wait(0) + // ensure no unexpected errors + expect(errors).toHaveLength(expectErrors) + if (publisher) { + expect(publisher.onError).toHaveBeenCalledTimes(expectErrors) + } + }) + + afterEach(async () => { + await wait(0) + if (publisher) { + publisher.debug('disconnecting after test') + await publisher.disconnect() + } + + if (subscriber) { + subscriber.debug('disconnecting after test') + await subscriber.disconnect() + } + + const openSockets = Connection.getOpen() + if (openSockets !== 0) { + await Connection.closeOpen() + throw new Error(`sockets not closed: ${openSockets}`) + } + }) + + let publisherPrivateKey: string + let subscriberPrivateKey: string + + async function setupPublisher(opts?: any) { + const client = createClient(opts) + await Promise.all([ + client.session.getSessionToken(), + client.connect(), + ]) + + const name = uid('stream') + stream = await client.createStream({ + name, + requireEncryptedData: true, + }) + + await stream.addToStorageNode(config.clientOptions.storageNode.address) + + publishTestMessages = getPublishTestMessages(client, { + stream, + waitForLast: true, + }) + return client + } + + beforeEach(async () => { + publisherPrivateKey = fakePrivateKey() + publisher = await setupPublisher({ + id: 'publisher', + auth: { + privateKey: publisherPrivateKey, + } + }) + subscriberPrivateKey = fakePrivateKey() + subscriber = createClient({ + id: 'subscriber', + autoConnect: true, + autoDisconnect: true, + auth: { + privateKey: subscriberPrivateKey, + } + }) + const otherUser = await subscriber.getUserInfo() + await stream.grantPermission('stream_get', otherUser.username) + await stream.grantPermission('stream_subscribe', otherUser.username) + const groupKey = GroupKey.generate() + await publisher.setNextGroupKey(stream.id, groupKey) + }) + + it('publisher persists group key', async () => { + // ensure publisher can read a persisted group key + // 1. publish some messages with publisher + // 2. then disconnect publisher + // 3. create new publisher with same key + // 4. subscribe with subscriber + // because original publisher is disconnected + // subscriber will need to ask new publisher + // for group keys, which the new publisher will have to read from + // persistence + const published = await publishTestMessages(5) + await publisher.disconnect() + const publisher2 = createClient({ + id: 'publisher2', + auth: { + privateKey: publisherPrivateKey, + } + }) + + addAfter(async () => { + await publisher2.disconnect() + }) + + await publisher2.connect() + + // TODO: this should probably happen automatically if there are keys + // also probably needs to create a connection handle + await publisher2.publisher.startKeyExchange() + + const sub = await subscriber.subscribe({ + stream: stream.id, + resend: { + last: 5, + } + }) + const received = [] + for await (const m of sub) { + received.push(m.getParsedContent()) + if (received.length === published.length) { + break + } + } + + expect(received).toEqual(published) + }, 2 * TIMEOUT) + + it('subscriber persists group key', async () => { + // we want to check that subscriber can read a group key + // persisted by another subscriber: + // 1. create publisher and subscriber + // 2. after subscriber gets first message + // 3. disconnect both subscriber and publisher + // 4. then create a new subscriber with same key as original subscriber + // 5. and subscribe to same stream. + // this should pick up group key persisted by first subscriber + // publisher is disconnected, so can't ask for new group keys + const sub = await subscriber.subscribe({ + stream: stream.id, + }) + const published = await publishTestMessages(5) + + const received = [] + for await (const m of sub) { + received.push(m.getParsedContent()) + if (received.length === 1) { + break + } + } + await subscriber.disconnect() + await publisher.disconnect() + + const subscriber2 = createClient({ + id: 'subscriber2', + auth: { + privateKey: subscriberPrivateKey + } + }) + + addAfter(async () => { + await subscriber2.disconnect() + }) + + await subscriber2.connect() + const sub2 = await subscriber2.subscribe({ + stream: stream.id, + resend: { + last: 5 + } + }) + + const received2 = [] + for await (const m of sub2) { + received2.push(m.getParsedContent()) + if (received2.length === published.length) { + break + } + } + expect(received2).toEqual(published) + expect(received).toEqual(published.slice(0, 1)) + }, 2 * TIMEOUT) + + it('can run multiple publishers in parallel', async () => { + const sub = await subscriber.subscribe({ + stream: stream.id, + }) + + // ensure publishers don't clobber each others data + const publisher2 = createClient({ + id: 'publisher2', + auth: { + privateKey: publisherPrivateKey, + } + }) + + addAfter(async () => { + await publisher2.disconnect() + }) + + await publisher2.connect() + const publishTestMessages2 = getPublishTestMessages(publisher2, { + stream, + waitForLast: true, + }) + + const [published1, published2] = await Promise.all([ + publishTestMessages(5), + publishTestMessages2(6), // use different lengths so we can differentiate who published what + ]) + + const received1 = [] + const received2 = [] + for await (const m of sub) { + const content = m.getParsedContent() + // 'n of 6' messages belong to publisher2 + if (content.value.endsWith('of 6')) { + received2.push(content) + } else { + received1.push(content) + } + + if (received1.length === published1.length && received2.length === published2.length) { + break + } + } + + expect(received1).toEqual(published1) + expect(received2).toEqual(published2) + }, 2 * TIMEOUT) +}) + From 703ef730845fc6dde4855e3957244a6f1d7a2dfc Mon Sep 17 00:00:00 2001 From: Tim Oxley Date: Wed, 5 May 2021 15:09:41 -0400 Subject: [PATCH 15/28] test(keyexchange): Fix conflicts from NET-208-revocation branch. --- src/publish/Encrypt.ts | 5 ++ src/stream/KeyExchange.ts | 84 +++++++++---------- .../EncryptionKeyPersistence.test.ts | 26 +++--- 3 files changed, 59 insertions(+), 56 deletions(-) diff --git a/src/publish/Encrypt.ts b/src/publish/Encrypt.ts index f9cb8fca1..839c48e3a 100644 --- a/src/publish/Encrypt.ts +++ b/src/publish/Encrypt.ts @@ -49,7 +49,12 @@ export default function Encrypt(client: StreamrClient) { if (streamMessage.messageType !== StreamMessage.MESSAGE_TYPES.MESSAGE) { return } + const [groupKey, nextGroupKey] = await getPublisherKeyExchange().useGroupKey(stream.id) + if (!groupKey) { + throw new Error(`Tried to use group key but no group key found for stream: ${stream.id}`) + } + await EncryptionUtil.encryptStreamMessage(streamMessage, groupKey, nextGroupKey) } diff --git a/src/stream/KeyExchange.ts b/src/stream/KeyExchange.ts index 1f0cf340b..d7115f034 100644 --- a/src/stream/KeyExchange.ts +++ b/src/stream/KeyExchange.ts @@ -108,49 +108,48 @@ function GroupKeyStore({ clientId, streamId, groupKeys }: GroupKeyStoreOptions) async has(id: GroupKeyId) { if (currentGroupKeyId === id) { return true } - if (nextGroupKeys.some((nextKey) => nextKey.id === id)) { return true } + if (nextGroupKeys.some((nextKey) => nextKey.id === id)) { return true } return store.has(id) }, async isEmpty() { return !nextGroupKeys.length && await store.size() === 0 }, - async useGroupKey(): Promise<[GroupKey, GroupKey | undefined]> { + async useGroupKey(): Promise<[GroupKey | undefined, GroupKey | undefined]> { const nextGroupKey = nextGroupKeys.pop() - switch (true) { - // First use of group key on this stream, no current key. Make next key current. - case !!(!currentGroupKeyId && nextGroupKey): { - await storeKey(nextGroupKey) - currentGroupKeyId = nextGroupKey.id - return [ - await this.get(currentGroupKeyId), - undefined, - ] - } - // Keep using current key (empty next) - case !!(currentGroupKeyId && !nextGroupKey): { - return [ - await this.get(currentGroupKeyId), - undefined - ] - } - // Key changed (non-empty next). return current + next. Make next key current. - case !!(currentGroupKeyId && nextGroupKey): { - await storeKey(nextGroupKey) - const prevGroupKey = await this.get(currentGroupKeyId) - currentGroupKeyId = nextGroupKey.id - // use current key one more time - return [ - prevGroupKey, - nextGroupKey, - ] - } - // Generate & use new key if none already set. - default: { - await this.rotateGroupKey() - return this.useGroupKey() + // First use of group key on this stream, no current key. Make next key current. + if (!currentGroupKeyId && nextGroupKey) { + await storeKey(nextGroupKey) + currentGroupKeyId = nextGroupKey.id + return [ + await this.get(currentGroupKeyId), + undefined, + ] } - } + + // Keep using current key (empty next) + if (currentGroupKeyId != null && !nextGroupKey) { + return [ + await this.get(currentGroupKeyId), + undefined + ] + } + + // Key changed (non-empty next). return current + next. Make next key current. + if (currentGroupKeyId != null && nextGroupKey != null) { + await storeKey(nextGroupKey) + const prevGroupKey = await this.get(currentGroupKeyId) + currentGroupKeyId = nextGroupKey.id + // use current key one more time + return [ + prevGroupKey, + nextGroupKey, + ] + } + + // Generate & use new key if none already set. + await this.rotateGroupKey() + return this.useGroupKey() }, async get(id: GroupKeyId) { return store.get(id) @@ -257,7 +256,7 @@ async function PublisherKeyExhangeSubscription(client: StreamrClient, getGroupKe const subscriberId = streamMessage.getPublisherId() const groupKeyStore = await getGroupKeyStore(streamId) - const isSubscriber = await client.isStreamSubscriber(streamId, subscriberId) + const isSubscriber = await client.isStreamSubscriber(streamId, subscriberId) const encryptedGroupKeys = (!isSubscriber ? [] : await Promise.all(groupKeyIds.map(async (id) => { const groupKey = await groupKeyStore.get(id) if (!groupKey) { @@ -357,7 +356,7 @@ export function PublisherKeyExhange(client: StreamrClient, { groupKeys = {} }: K async function useGroupKey(streamId: string) { await next() - if (!enabled) { return undefined } + if (!enabled) { return [] } const groupKeyStore = await getGroupKeyStore(streamId) return groupKeyStore.useGroupKey() } @@ -367,10 +366,10 @@ export function PublisherKeyExhange(client: StreamrClient, { groupKeys = {} }: K return !groupKeyStore.isEmpty() } - async function rekey(streamId) { + async function rekey(streamId: string) { if (!enabled) { return } - const groupKeyStore = getGroupKeyStore(streamId) - groupKeyStore.rekey() + const groupKeyStore = await getGroupKeyStore(streamId) + await groupKeyStore.rekey() await next() } @@ -652,11 +651,12 @@ export function SubscriberKeyExchange(client: StreamrClient, { groupKeys = {} }: enabled = true return next() }, - async addNewKey(streamMessage) { + async addNewKey(streamMessage: StreamMessage) { if (!streamMessage.newGroupKey) { return } const streamId = streamMessage.getStreamId() const groupKeyStore = await getGroupKeyStore(streamId) - return groupKeyStore.add(streamMessage.newGroupKey) + const newGroupKey: unknown = streamMessage.newGroupKey + await groupKeyStore.add(newGroupKey as GroupKey) }, async stop() { enabled = false diff --git a/test/integration/EncryptionKeyPersistence.test.ts b/test/integration/EncryptionKeyPersistence.test.ts index 0596b34eb..c105f4ed8 100644 --- a/test/integration/EncryptionKeyPersistence.test.ts +++ b/test/integration/EncryptionKeyPersistence.test.ts @@ -1,29 +1,28 @@ import { wait } from 'streamr-test-utils' import { describeRepeats, fakePrivateKey, uid, getPublishTestMessages, addAfterFn } from '../utils' -// import { Defer } from '../../src/utils' import { StreamrClient } from '../../src/StreamrClient' +import { Stream, StreamOperation } from '../../src/stream' import { GroupKey } from '../../src/stream/Encryption' import Connection from '../../src/Connection' import config from './config' -const TIMEOUT = 30 * 1000 +const TIMEOUT = 10 * 1000 describeRepeats('decryption', () => { - let publishTestMessages let expectErrors = 0 // check no errors by default - let errors = [] - let subscriber - const addAfter = addAfterFn() - - const getOnError = (errs) => jest.fn((err) => { + let errors: Error[] = [] + let onError = jest.fn() + const getOnError = (errs: Error[]) => jest.fn((err) => { errs.push(err) }) - let onError = jest.fn() - let publisher - let stream + let publisher: StreamrClient + let subscriber: StreamrClient + let stream: Stream + let publishTestMessages: ReturnType + const addAfter = addAfterFn() const createClient = (opts = {}) => { const c = new StreamrClient({ @@ -122,8 +121,8 @@ describeRepeats('decryption', () => { } }) const otherUser = await subscriber.getUserInfo() - await stream.grantPermission('stream_get', otherUser.username) - await stream.grantPermission('stream_subscribe', otherUser.username) + await stream.grantPermission(StreamOperation.STREAM_GET, otherUser.username) + await stream.grantPermission(StreamOperation.STREAM_SUBSCRIBE, otherUser.username) const groupKey = GroupKey.generate() await publisher.setNextGroupKey(stream.id, groupKey) }) @@ -277,4 +276,3 @@ describeRepeats('decryption', () => { expect(received2).toEqual(published2) }, 2 * TIMEOUT) }) - From adf864ecdac9d9898e5312ee779f19402a3cae0d Mon Sep 17 00:00:00 2001 From: Tim Oxley Date: Wed, 5 May 2021 15:10:37 -0400 Subject: [PATCH 16/28] feat(keyexchange): Use sqlite for serverside persistent storage. --- package-lock.json | 559 ++------------------------------ package.json | 3 - src/stream/PersistentStore.ts | 87 ++--- test/unit/GroupKeyStore.test.ts | 147 +++++++++ 4 files changed, 217 insertions(+), 579 deletions(-) create mode 100644 test/unit/GroupKeyStore.test.ts diff --git a/package-lock.json b/package-lock.json index 2fb69011c..cf7903b69 100644 --- a/package-lock.json +++ b/package-lock.json @@ -3012,11 +3012,6 @@ "integrity": "sha512-dgasobK/Y0wVMswcipr3k0HpevxFJLijN03A8mYfEPvWvOs14v0ZlYTR4kIgMx8g4+fTyTFv8/jLCIfRqLDJ4A==", "dev": true }, - "@types/abstract-leveldown": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/@types/abstract-leveldown/-/abstract-leveldown-5.0.1.tgz", - "integrity": "sha512-wYxU3kp5zItbxKmeRYCEplS2MW7DzyBnxPGj+GJVHZEUZiK/nn5Ei1sUFgURDh+X051+zsGe28iud3oHjrYWQQ==" - }, "@types/asn1js": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/@types/asn1js/-/asn1js-2.0.0.tgz", @@ -3087,15 +3082,6 @@ "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.5.tgz", "integrity": "sha512-Q1y515GcOdTHgagaVFhHnIFQ38ygs/kmxdNpvpou+raI9UO3YZcHDngBSYKQklcKlvA7iuQlmIKbzvmxcOE9CQ==" }, - "@types/encoding-down": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/@types/encoding-down/-/encoding-down-5.0.0.tgz", - "integrity": "sha512-G0MlS/+/U2RIQLcSEhhAcoMrXw3hXUCFSKbhbeEljoKMra2kq+NPX6tfOveSWQLX2hJXBo+YrvKgAGe+tFL1Aw==", - "requires": { - "@types/abstract-leveldown": "*", - "@types/level-codec": "*" - } - }, "@types/eslint": { "version": "7.2.10", "resolved": "https://registry.npmjs.org/@types/eslint/-/eslint-7.2.10.tgz", @@ -3200,30 +3186,6 @@ "integrity": "sha1-7ihweulOEdK4J7y+UnC86n8+ce4=", "dev": true }, - "@types/level": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/@types/level/-/level-6.0.0.tgz", - "integrity": "sha512-NjaUpukKfCvnV4Wk0jUaodFi2/66HxgpYghc2aV8iP+zk2NMt/9ps1eVlifqOU/+eLzMlDIY69NWkbPaAstukQ==", - "requires": { - "@types/abstract-leveldown": "*", - "@types/encoding-down": "*", - "@types/levelup": "*" - } - }, - "@types/level-codec": { - "version": "9.0.0", - "resolved": "https://registry.npmjs.org/@types/level-codec/-/level-codec-9.0.0.tgz", - "integrity": "sha512-SWYkVJylo1dqblkhrr7UtmsQh4wdZA9bV1y3QJSywMPSqGfW0p1w37N1EayZtKbg1dGReIIQEEOtxk4wZvGrWQ==" - }, - "@types/levelup": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/@types/levelup/-/levelup-4.3.1.tgz", - "integrity": "sha512-n//PeTpbHLjMLTIgW5B/g06W/6iuTBHuvUka2nFL9APMSVMNe2r4enADfu3CIE9IyV9E+uquf9OEQQqrDeg24A==", - "requires": { - "@types/abstract-leveldown": "*", - "@types/node": "*" - } - }, "@types/lodash": { "version": "4.14.168", "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.168.tgz", @@ -3281,7 +3243,8 @@ "@types/node": { "version": "14.14.43", "resolved": "https://registry.npmjs.org/@types/node/-/node-14.14.43.tgz", - "integrity": "sha512-3pwDJjp1PWacPTpH0LcfhgjvurQvrZFBrC6xxjaUEZ7ifUtT32jtjPxEMMblpqd2Mvx+k8haqQJLQxolyGN/cQ==" + "integrity": "sha512-3pwDJjp1PWacPTpH0LcfhgjvurQvrZFBrC6xxjaUEZ7ifUtT32jtjPxEMMblpqd2Mvx+k8haqQJLQxolyGN/cQ==", + "dev": true }, "@types/node-fetch": { "version": "2.5.10", @@ -3712,29 +3675,6 @@ "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==" }, - "abstract-leveldown": { - "version": "6.2.3", - "resolved": "https://registry.npmjs.org/abstract-leveldown/-/abstract-leveldown-6.2.3.tgz", - "integrity": "sha512-BsLm5vFMRUrrLeCcRc+G0t2qOaTzpoJQLOubq2XM72eNpjF5UdU5o/5NvlNhx95XHcAvcl8OMXr4mlg/fRgUXQ==", - "requires": { - "buffer": "^5.5.0", - "immediate": "^3.2.3", - "level-concat-iterator": "~2.0.0", - "level-supports": "~1.0.0", - "xtend": "~4.0.0" - }, - "dependencies": { - "buffer": { - "version": "5.7.1", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", - "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", - "requires": { - "base64-js": "^1.3.1", - "ieee754": "^1.1.13" - } - } - } - }, "accepts": { "version": "1.3.7", "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.7.tgz", @@ -5007,11 +4947,6 @@ "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz", "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=" }, - "catering": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/catering/-/catering-2.0.0.tgz", - "integrity": "sha512-aD/WmxhGwUGsVPrj8C80vH7C7GphJilYVSdudoV4u16XdrLF7CVyfBmENsc4tLTVsJJzCRid8GbwJ7mcPLee6Q==" - }, "chai-nightwatch": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/chai-nightwatch/-/chai-nightwatch-0.4.0.tgz", @@ -5785,15 +5720,6 @@ "clone": "^1.0.2" } }, - "deferred-leveldown": { - "version": "5.3.0", - "resolved": "https://registry.npmjs.org/deferred-leveldown/-/deferred-leveldown-5.3.0.tgz", - "integrity": "sha512-a59VOT+oDy7vtAbLRCZwWgxu2BaCfd5Hk7wxJd48ei7I+nsg8Orlb9CLG0PMZienk9BSUKgeAqkO2+Lw+1+Ukw==", - "requires": { - "abstract-leveldown": "~6.2.1", - "inherits": "^2.0.3" - } - }, "define-properties": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.3.tgz", @@ -6033,17 +5959,6 @@ } } }, - "duplexify": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-4.1.1.tgz", - "integrity": "sha512-DY3xVEmVHTv1wSzKNbwoU6nVjzI369Y6sPoqfYr0/xlx3IdX2n94xIszTcjPO8W8ZIv0Wb0PXNcjuZyT4wiICA==", - "requires": { - "end-of-stream": "^1.4.1", - "inherits": "^2.0.3", - "readable-stream": "^3.1.1", - "stream-shift": "^1.0.0" - } - }, "ecc-jsbn": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz", @@ -6112,21 +6027,11 @@ "integrity": "sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k=", "dev": true }, - "encoding-down": { - "version": "6.3.0", - "resolved": "https://registry.npmjs.org/encoding-down/-/encoding-down-6.3.0.tgz", - "integrity": "sha512-QKrV0iKR6MZVJV08QY0wp1e7vF6QbhnbQhb07bwpEyuz4uZiZgPlEGdkCROuFkUwdxlFaiPIhjyarH1ee/3vhw==", - "requires": { - "abstract-leveldown": "^6.2.1", - "inherits": "^2.0.3", - "level-codec": "^9.0.0", - "level-errors": "^2.0.0" - } - }, "end-of-stream": { "version": "1.4.4", "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", + "dev": true, "requires": { "once": "^1.4.0" } @@ -6166,6 +6071,7 @@ "version": "0.1.8", "resolved": "https://registry.npmjs.org/errno/-/errno-0.1.8.tgz", "integrity": "sha512-dJ6oBr5SQ1VSd9qkk7ByRgb/1SH4JZjCHSW/mr63/QcXO9zLVxvJ6Oy13nio03rxpSnVDDjFor75SjVeZWPW/A==", + "dev": true, "requires": { "prr": "~1.0.1" } @@ -8183,11 +8089,6 @@ "minimatch": "^3.0.4" } }, - "immediate": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/immediate/-/immediate-3.3.0.tgz", - "integrity": "sha512-HR7EVodfFUdQCTIeySw+WDRFJlPcLOJbXfwwZ7Oom6tjsvZ3bOkCDJHehQC3nxJrv7+f9XecwazynjU8e4Vw3Q==" - }, "import-fresh": { "version": "3.3.0", "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.0.tgz", @@ -10439,352 +10340,6 @@ "integrity": "sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w==", "dev": true }, - "length-prefixed-stream": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/length-prefixed-stream/-/length-prefixed-stream-2.0.0.tgz", - "integrity": "sha512-dvjTuWTKWe0oEznQcG6a9osfiYknCs7DEFJMP88n9Y581IFhYh1sZIgAFcuDOojKB0G7ftPreKhh4D0kh/VPjQ==", - "requires": { - "inherits": "^2.0.3", - "readable-stream": "^3.1.1", - "varint": "^5.0.0" - } - }, - "level": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/level/-/level-7.0.0.tgz", - "integrity": "sha512-QrBnjcWywalh86ms9hfizvxT5aBHrgWEu6rLChS9tFE2wwFU3aI1r0v+2SSZIyeUr4O4PFo8+sCc1kebahdhlw==", - "requires": { - "level-js": "^6.0.0", - "level-packager": "^6.0.0", - "leveldown": "^6.0.0" - } - }, - "level-codec": { - "version": "9.0.2", - "resolved": "https://registry.npmjs.org/level-codec/-/level-codec-9.0.2.tgz", - "integrity": "sha512-UyIwNb1lJBChJnGfjmO0OR+ezh2iVu1Kas3nvBS/BzGnx79dv6g7unpKIDNPMhfdTEGoc7mC8uAu51XEtX+FHQ==", - "requires": { - "buffer": "^5.6.0" - }, - "dependencies": { - "buffer": { - "version": "5.7.1", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", - "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", - "requires": { - "base64-js": "^1.3.1", - "ieee754": "^1.1.13" - } - } - } - }, - "level-concat-iterator": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/level-concat-iterator/-/level-concat-iterator-2.0.1.tgz", - "integrity": "sha512-OTKKOqeav2QWcERMJR7IS9CUo1sHnke2C0gkSmcR7QuEtFNLLzHQAvnMw8ykvEcv0Qtkg0p7FOwP1v9e5Smdcw==" - }, - "level-errors": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/level-errors/-/level-errors-2.0.1.tgz", - "integrity": "sha512-UVprBJXite4gPS+3VznfgDSU8PTRuVX0NXwoWW50KLxd2yw4Y1t2JUR5In1itQnudZqRMT9DlAM3Q//9NCjCFw==", - "requires": { - "errno": "~0.1.1" - } - }, - "level-iterator-stream": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/level-iterator-stream/-/level-iterator-stream-4.0.2.tgz", - "integrity": "sha512-ZSthfEqzGSOMWoUGhTXdX9jv26d32XJuHz/5YnuHZzH6wldfWMOVwI9TBtKcya4BKTyTt3XVA0A3cF3q5CY30Q==", - "requires": { - "inherits": "^2.0.4", - "readable-stream": "^3.4.0", - "xtend": "^4.0.2" - } - }, - "level-js": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/level-js/-/level-js-6.0.0.tgz", - "integrity": "sha512-7dp7JuaoQoqKW4ZGvrV1RB5f51/ktLdEo9fSDsh3Ofmg7sKCMu3X0CIngbY/IUz/YyskhN7LRvEVIkZHCY3LKQ==", - "requires": { - "abstract-leveldown": "^7.0.0", - "buffer": "^6.0.3", - "inherits": "^2.0.3", - "ltgt": "^2.1.2" - }, - "dependencies": { - "abstract-leveldown": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/abstract-leveldown/-/abstract-leveldown-7.0.0.tgz", - "integrity": "sha512-mFAi5sB/UjpNYglrQ4irzdmr2mbQtE94OJbrAYuK2yRARjH/OACinN1meOAorfnaLPMQdFymSQMlkiDm9AXXKQ==", - "requires": { - "buffer": "^6.0.3", - "is-buffer": "^2.0.5", - "level-concat-iterator": "^3.0.0", - "level-supports": "^2.0.0", - "queue-microtask": "^1.2.3" - } - }, - "is-buffer": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-2.0.5.tgz", - "integrity": "sha512-i2R6zNFDwgEHJyQUtJEk0XFi1i0dPFn/oqjK3/vPCcDeJvW5NQ83V8QbicfF1SupOaB0h8ntgBC2YiE7dfyctQ==" - }, - "level-concat-iterator": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/level-concat-iterator/-/level-concat-iterator-3.0.0.tgz", - "integrity": "sha512-UHGiIdj+uiFQorOrURRvJF3Ei0uHc89ciM/aRi0qsWDV2f0HXypeXUPhJKL6DsONgSR76Pc0AI4sKYEYYRn2Dg==" - }, - "level-supports": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/level-supports/-/level-supports-2.0.0.tgz", - "integrity": "sha512-8UJgzo1pvWP1wq80ZlkL19fPeK7tlyy0sBY90+2pj0x/kvzHCoLDWyuFJJMrsTn33oc7hbMkS3SkjCxMRPHWaw==" - }, - "queue-microtask": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", - "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==" - } - } - }, - "level-packager": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/level-packager/-/level-packager-6.0.0.tgz", - "integrity": "sha512-me656XRWfOVqs9wc+mWckZ6Rb1GuP33ndN4ZntDXwXFspX8cGA++Y+YqJsdE/mjTiipTuxXf047Z4rV62nOVuw==", - "requires": { - "encoding-down": "^7.0.0", - "levelup": "^5.0.0" - }, - "dependencies": { - "abstract-leveldown": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/abstract-leveldown/-/abstract-leveldown-7.0.0.tgz", - "integrity": "sha512-mFAi5sB/UjpNYglrQ4irzdmr2mbQtE94OJbrAYuK2yRARjH/OACinN1meOAorfnaLPMQdFymSQMlkiDm9AXXKQ==", - "requires": { - "buffer": "^6.0.3", - "is-buffer": "^2.0.5", - "level-concat-iterator": "^3.0.0", - "level-supports": "^2.0.0", - "queue-microtask": "^1.2.3" - } - }, - "deferred-leveldown": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/deferred-leveldown/-/deferred-leveldown-6.0.0.tgz", - "integrity": "sha512-F6CLAZzNeURojlH4MCigZr54tNz+xDSi06YXsDr5uLSKeF3JKnvnQWTqd+RETh2hbWTJR3qDzGicQOWS5ZQ1BQ==", - "requires": { - "abstract-leveldown": "^7.0.0", - "inherits": "^2.0.3" - } - }, - "encoding-down": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/encoding-down/-/encoding-down-7.0.0.tgz", - "integrity": "sha512-hor6z2W/ZrVqDYMawQp7VtfEt6BrvYw+mgjWLauUMZsIBjMt1/k5aa+JreLbtjwJdkjrZ39TU+pV5GpHPGRpog==", - "requires": { - "abstract-leveldown": "^7.0.0", - "inherits": "^2.0.3", - "level-codec": "^10.0.0", - "level-errors": "^3.0.0" - } - }, - "errno": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/errno/-/errno-1.0.0.tgz", - "integrity": "sha512-3zV5mFS1E8/1bPxt/B0xxzI1snsg3uSCIh6Zo1qKg6iMw93hzPANk9oBFzSFBFrwuVoQuE3rLoouAUfwOAj1wQ==", - "requires": { - "prr": "~1.0.1" - } - }, - "is-buffer": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-2.0.5.tgz", - "integrity": "sha512-i2R6zNFDwgEHJyQUtJEk0XFi1i0dPFn/oqjK3/vPCcDeJvW5NQ83V8QbicfF1SupOaB0h8ntgBC2YiE7dfyctQ==" - }, - "level-codec": { - "version": "10.0.0", - "resolved": "https://registry.npmjs.org/level-codec/-/level-codec-10.0.0.tgz", - "integrity": "sha512-QW3VteVNAp6c/LuV6nDjg7XDXx9XHK4abmQarxZmlRSDyXYk20UdaJTSX6yzVvQ4i0JyWSB7jert0DsyD/kk6g==", - "requires": { - "buffer": "^6.0.3" - } - }, - "level-concat-iterator": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/level-concat-iterator/-/level-concat-iterator-3.0.0.tgz", - "integrity": "sha512-UHGiIdj+uiFQorOrURRvJF3Ei0uHc89ciM/aRi0qsWDV2f0HXypeXUPhJKL6DsONgSR76Pc0AI4sKYEYYRn2Dg==" - }, - "level-errors": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/level-errors/-/level-errors-3.0.0.tgz", - "integrity": "sha512-MZXOQT061uEjxxxq4C/Jf+M3RdEKK9e3NbxlN7yOp1LDYoLVAhE2i1j0b7XqXfl8FjFtUL7phwr3Sn0wXXoMqA==", - "requires": { - "errno": "^1.0.0" - } - }, - "level-iterator-stream": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/level-iterator-stream/-/level-iterator-stream-5.0.0.tgz", - "integrity": "sha512-wnb1+o+CVFUDdiSMR/ZymE2prPs3cjVLlXuDeSq9Zb8o032XrabGEXcTCsBxprAtseO3qvFeGzh6406z9sOTRA==", - "requires": { - "inherits": "^2.0.4", - "readable-stream": "^3.4.0" - } - }, - "level-supports": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/level-supports/-/level-supports-2.0.0.tgz", - "integrity": "sha512-8UJgzo1pvWP1wq80ZlkL19fPeK7tlyy0sBY90+2pj0x/kvzHCoLDWyuFJJMrsTn33oc7hbMkS3SkjCxMRPHWaw==" - }, - "levelup": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/levelup/-/levelup-5.0.0.tgz", - "integrity": "sha512-P4IKS4J17b6dzm8iI8Irv5gvOlrqJv04Lrpq1rAgZvjR2IsVSjbXQQo1LoK/PJuouxepLE8CTIiKGmHQYZnneA==", - "requires": { - "catering": "^2.0.0", - "deferred-leveldown": "^6.0.0", - "level-errors": "^3.0.0", - "level-iterator-stream": "^5.0.0", - "level-supports": "^2.0.0", - "queue-microtask": "^1.2.3" - } - }, - "queue-microtask": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", - "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==" - } - } - }, - "level-party": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/level-party/-/level-party-5.0.0.tgz", - "integrity": "sha512-wtvzpTTDv5CB/Wca+uaDmIFqMesY7NhkPLVSRtsx6DMvxpP4vvrIlRVAthLnysdpUNYcZvtSbLqRDFkwu0y+aw==", - "requires": { - "level": "^6.0.0", - "multileveldown": "^3.0.0", - "readable-stream": "^3.6.0" - }, - "dependencies": { - "buffer": { - "version": "5.7.1", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", - "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", - "requires": { - "base64-js": "^1.3.1", - "ieee754": "^1.1.13" - } - }, - "level": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/level/-/level-6.0.1.tgz", - "integrity": "sha512-psRSqJZCsC/irNhfHzrVZbmPYXDcEYhA5TVNwr+V92jF44rbf86hqGp8fiT702FyiArScYIlPSBTDUASCVNSpw==", - "requires": { - "level-js": "^5.0.0", - "level-packager": "^5.1.0", - "leveldown": "^5.4.0" - } - }, - "level-js": { - "version": "5.0.2", - "resolved": "https://registry.npmjs.org/level-js/-/level-js-5.0.2.tgz", - "integrity": "sha512-SnBIDo2pdO5VXh02ZmtAyPP6/+6YTJg2ibLtl9C34pWvmtMEmRTWpra+qO/hifkUtBTOtfx6S9vLDjBsBK4gRg==", - "requires": { - "abstract-leveldown": "~6.2.3", - "buffer": "^5.5.0", - "inherits": "^2.0.3", - "ltgt": "^2.1.2" - } - }, - "level-packager": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/level-packager/-/level-packager-5.1.1.tgz", - "integrity": "sha512-HMwMaQPlTC1IlcwT3+swhqf/NUO+ZhXVz6TY1zZIIZlIR0YSn8GtAAWmIvKjNY16ZkEg/JcpAuQskxsXqC0yOQ==", - "requires": { - "encoding-down": "^6.3.0", - "levelup": "^4.3.2" - } - }, - "leveldown": { - "version": "5.6.0", - "resolved": "https://registry.npmjs.org/leveldown/-/leveldown-5.6.0.tgz", - "integrity": "sha512-iB8O/7Db9lPaITU1aA2txU/cBEXAt4vWwKQRrrWuS6XDgbP4QZGj9BL2aNbwb002atoQ/lIotJkfyzz+ygQnUQ==", - "requires": { - "abstract-leveldown": "~6.2.1", - "napi-macros": "~2.0.0", - "node-gyp-build": "~4.1.0" - } - }, - "node-gyp-build": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.1.1.tgz", - "integrity": "sha512-dSq1xmcPDKPZ2EED2S6zw/b9NKsqzXRE6dVr8TVQnI3FJOTteUMuqF3Qqs6LZg+mLGYJWqQzMbIjMtJqTv87nQ==" - } - } - }, - "level-supports": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/level-supports/-/level-supports-1.0.1.tgz", - "integrity": "sha512-rXM7GYnW8gsl1vedTJIbzOrRv85c/2uCMpiiCzO2fndd06U/kUXEEU9evYn4zFggBOg36IsBW8LzqIpETwwQzg==", - "requires": { - "xtend": "^4.0.2" - } - }, - "leveldown": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/leveldown/-/leveldown-6.0.0.tgz", - "integrity": "sha512-NEsyqpfdDhpFO49Zm9htNSsWixMa9Q9sUXgrBTaQNPyPo2Kx1wRctgIXMzc7tduXJqNff8QAwulv2eZDboghxQ==", - "requires": { - "abstract-leveldown": "^7.0.0", - "napi-macros": "~2.0.0", - "node-gyp-build": "~4.2.1" - }, - "dependencies": { - "abstract-leveldown": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/abstract-leveldown/-/abstract-leveldown-7.0.0.tgz", - "integrity": "sha512-mFAi5sB/UjpNYglrQ4irzdmr2mbQtE94OJbrAYuK2yRARjH/OACinN1meOAorfnaLPMQdFymSQMlkiDm9AXXKQ==", - "requires": { - "buffer": "^6.0.3", - "is-buffer": "^2.0.5", - "level-concat-iterator": "^3.0.0", - "level-supports": "^2.0.0", - "queue-microtask": "^1.2.3" - } - }, - "is-buffer": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-2.0.5.tgz", - "integrity": "sha512-i2R6zNFDwgEHJyQUtJEk0XFi1i0dPFn/oqjK3/vPCcDeJvW5NQ83V8QbicfF1SupOaB0h8ntgBC2YiE7dfyctQ==" - }, - "level-concat-iterator": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/level-concat-iterator/-/level-concat-iterator-3.0.0.tgz", - "integrity": "sha512-UHGiIdj+uiFQorOrURRvJF3Ei0uHc89ciM/aRi0qsWDV2f0HXypeXUPhJKL6DsONgSR76Pc0AI4sKYEYYRn2Dg==" - }, - "level-supports": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/level-supports/-/level-supports-2.0.0.tgz", - "integrity": "sha512-8UJgzo1pvWP1wq80ZlkL19fPeK7tlyy0sBY90+2pj0x/kvzHCoLDWyuFJJMrsTn33oc7hbMkS3SkjCxMRPHWaw==" - }, - "queue-microtask": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", - "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==" - } - } - }, - "levelup": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/levelup/-/levelup-4.4.0.tgz", - "integrity": "sha512-94++VFO3qN95cM/d6eBXvd894oJE0w3cInq9USsyQzzoJxmiYzPAocNcuGCPGGjoXqDVJcr3C1jzt1TSjyaiLQ==", - "requires": { - "deferred-leveldown": "~5.3.0", - "level-errors": "~2.0.0", - "level-iterator-stream": "~4.0.0", - "level-supports": "~1.0.0", - "xtend": "~4.0.0" - } - }, "leven": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/leven/-/leven-3.1.0.tgz", @@ -11059,11 +10614,6 @@ "yallist": "^4.0.0" } }, - "ltgt": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/ltgt/-/ltgt-2.2.1.tgz", - "integrity": "sha1-81ypHEk/e3PaDgdJUwTxezH4fuU=" - }, "lunr": { "version": "2.3.9", "resolved": "https://registry.npmjs.org/lunr/-/lunr-2.3.9.tgz", @@ -11765,22 +11315,6 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, - "multileveldown": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/multileveldown/-/multileveldown-3.0.0.tgz", - "integrity": "sha512-F1R1jotuSM0kD3G5HOGTzJtjv3oHLLkKX6Gy+3fjKxah0Ami1WBR2ZwURbXwo/01XXDfgyBP0ankc7Yn+boSwA==", - "requires": { - "abstract-leveldown": "^6.2.2", - "duplexify": "^4.1.1", - "encoding-down": "^6.3.0", - "end-of-stream": "^1.1.0", - "length-prefixed-stream": "^2.0.0", - "levelup": "^4.3.2", - "numeric-id-map": "^1.1.0", - "protocol-buffers-encodings": "^1.1.0", - "reachdown": "^1.0.0" - } - }, "mute-stream": { "version": "0.0.8", "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.8.tgz", @@ -11811,11 +11345,6 @@ "to-regex": "^3.0.1" } }, - "napi-macros": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/napi-macros/-/napi-macros-2.0.0.tgz", - "integrity": "sha512-A0xLykHtARfueITVDernsAWdtIMbOJgKgcluwENp3AlsKN/PloyO10HtmoqnFAQAcxPkgZN7wdfPfEd0zNGxbg==" - }, "natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", @@ -11994,15 +11523,6 @@ "minimist": "^1.2.5" } }, - "nopt": { - "version": "3.0.6", - "resolved": "https://registry.npmjs.org/nopt/-/nopt-3.0.6.tgz", - "integrity": "sha1-xkZdvwirzU2zWTF/eaxopkayj/k=", - "optional": true, - "requires": { - "abbrev": "1" - } - }, "rimraf": { "version": "2.7.1", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz", @@ -12139,6 +11659,15 @@ "minimist": "^1.2.5" } }, + "nopt": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/nopt/-/nopt-4.0.3.tgz", + "integrity": "sha512-CvaGwVMztSMJLOeXPrez7fyfObdZqNUK1cPAEzLHrTybIua9pMdmmPR5YwtfNftIOMv3DPUhFaxsZMNTQO20Kg==", + "requires": { + "abbrev": "1", + "osenv": "^0.1.4" + } + }, "rimraf": { "version": "2.7.1", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz", @@ -12193,12 +11722,12 @@ } }, "nopt": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/nopt/-/nopt-4.0.3.tgz", - "integrity": "sha512-CvaGwVMztSMJLOeXPrez7fyfObdZqNUK1cPAEzLHrTybIua9pMdmmPR5YwtfNftIOMv3DPUhFaxsZMNTQO20Kg==", + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/nopt/-/nopt-3.0.6.tgz", + "integrity": "sha1-xkZdvwirzU2zWTF/eaxopkayj/k=", + "optional": true, "requires": { - "abbrev": "1", - "osenv": "^0.1.4" + "abbrev": "1" } }, "normalize-package-data": { @@ -12283,11 +11812,6 @@ "resolved": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz", "integrity": "sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0=" }, - "numeric-id-map": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/numeric-id-map/-/numeric-id-map-1.1.0.tgz", - "integrity": "sha1-ERDkRFrmg+g9R56t0PbZJJ/aO9Y=" - }, "nwsapi": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/nwsapi/-/nwsapi-2.2.0.tgz", @@ -13037,22 +12561,6 @@ "sisteransi": "^1.0.5" } }, - "protocol-buffers-encodings": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/protocol-buffers-encodings/-/protocol-buffers-encodings-1.1.1.tgz", - "integrity": "sha512-5aFshI9SbhtcMiDiZZu3g2tMlZeS5lhni//AGJ7V34PQLU5JA91Cva7TIs6inZhYikS3OpnUzAUuL6YtS0CyDA==", - "requires": { - "signed-varint": "^2.0.1", - "varint": "5.0.0" - }, - "dependencies": { - "varint": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/varint/-/varint-5.0.0.tgz", - "integrity": "sha1-2Ca4n3SQcy+rwMDtaT7Uddyynr8=" - } - } - }, "proxy-addr": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.6.tgz", @@ -13105,7 +12613,8 @@ "prr": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/prr/-/prr-1.0.1.tgz", - "integrity": "sha1-0/wRS6BplaRexok/SEzrHXj19HY=" + "integrity": "sha1-0/wRS6BplaRexok/SEzrHXj19HY=", + "dev": true }, "psl": { "version": "1.8.0", @@ -13229,11 +12738,6 @@ } } }, - "reachdown": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/reachdown/-/reachdown-1.1.0.tgz", - "integrity": "sha512-6LsdRe4cZyOjw4NnvbhUd/rGG7WQ9HMopPr+kyL018Uci4kijtxcGR5kVb5Ln13k4PEE+fEFQbjfOvNw7cnXmA==" - }, "react-is": { "version": "17.0.1", "resolved": "https://registry.npmjs.org/react-is/-/react-is-17.0.1.tgz", @@ -14046,14 +13550,6 @@ "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.3.tgz", "integrity": "sha512-VUJ49FC8U1OxwZLxIbTTrDvLnf/6TDgxZcK8wxR8zs13xpx7xbG60ndBlhNrFi2EMuFRoeDoJO7wthSLq42EjA==" }, - "signed-varint": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/signed-varint/-/signed-varint-2.0.1.tgz", - "integrity": "sha1-UKmYnafJjCxh2tEZvJdHDvhSgSk=", - "requires": { - "varint": "~5.0.0" - } - }, "sinon": { "version": "9.2.4", "resolved": "https://registry.npmjs.org/sinon/-/sinon-9.2.4.tgz", @@ -14501,11 +13997,6 @@ "integrity": "sha1-NbCYdbT/SfJqd35QmzCQoyJr8ks=", "dev": true }, - "stream-shift": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.1.tgz", - "integrity": "sha512-AiisoFqQ0vbGcZgQPY1cdP2I76glaVA/RauYR4G4thNFgkTqr90yXTo4LYX60Jl+sIlPNHHdGSwo01AvbKUSVQ==" - }, "streamr-client-protocol": { "version": "8.0.2", "resolved": "https://registry.npmjs.org/streamr-client-protocol/-/streamr-client-protocol-8.0.2.tgz", @@ -15550,11 +15041,6 @@ "spdx-expression-parse": "^3.0.0" } }, - "varint": { - "version": "5.0.2", - "resolved": "https://registry.npmjs.org/varint/-/varint-5.0.2.tgz", - "integrity": "sha512-lKxKYG6H03yCZUpAGOPOsMcGxd1RHCu1iKvEHYDPmTyq2HueGhD73ssNBqqQWfvYs04G9iUFRvmAVLW20Jw6ow==" - }, "vary": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", @@ -16112,11 +15598,6 @@ "integrity": "sha1-UqY+VsoLhKfzpfPWGHLxJq16WUM=", "dev": true }, - "xtend": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", - "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==" - }, "y18n": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.1.tgz", diff --git a/package.json b/package.json index 5740e2aaf..2cc026710 100644 --- a/package.json +++ b/package.json @@ -150,12 +150,9 @@ "@ethersproject/transactions": "^5.1.1", "@ethersproject/wallet": "^5.1.0", "@ethersproject/web": "^5.1.0", - "@types/level": "^6.0.0", "debug": "^4.3.2", "env-paths": "^2.2.1", "eventemitter3": "^4.0.7", - "level": "^7.0.0", - "level-party": "^5.0.0", "lodash": "^4.17.21", "mem": "^8.1.1", "node-abort-controller": "^1.2.1", diff --git a/src/stream/PersistentStore.ts b/src/stream/PersistentStore.ts index c4282c977..935b76980 100644 --- a/src/stream/PersistentStore.ts +++ b/src/stream/PersistentStore.ts @@ -1,29 +1,25 @@ -import { once } from 'events' import envPaths from 'env-paths' -import Level from 'level' -// @ts-expect-error -import LevelParty from 'level-party' import { dirname, join } from 'path' import fs from 'fs/promises' +import { open, Database } from 'sqlite' +import sqlite3 from 'sqlite3' import { GroupKey } from './Encryption' import { pOnce } from '../utils' class ServerStorage { - readonly id: string + readonly clientId: string + readonly streamId: string readonly dbFilePath: string - private readonly store: Level.LevelDB + private store?: Database private error?: Error - constructor(id: string) { - this.id = encodeURIComponent(id) + constructor({ clientId, streamId }: { clientId: string, streamId: string }) { + this.streamId = encodeURIComponent(streamId) + this.clientId = encodeURIComponent(clientId) const paths = envPaths('streamr-client') - const dbFilePath = join(paths.data, `${id}.db`) + const dbFilePath = join(paths.data, clientId, 'GroupKeys.db') this.dbFilePath = dbFilePath - const Store = LevelParty as Level.Constructor - this.store = Store(dbFilePath, { valueEncoding: 'json' }, (err) => { - this.error = err - }) this.init = pOnce(this.init.bind(this)) } @@ -31,7 +27,18 @@ class ServerStorage { async init() { try { await fs.mkdir(dirname(this.dbFilePath), { recursive: true }) - await this.store.open() + // open the database + const store = await open({ + filename: this.dbFilePath, + driver: sqlite3.Database + }) + await store.exec(`CREATE TABLE IF NOT EXISTS GroupKeys ( + id TEXT, + groupKey TEXT, + streamId TEXT + )`) + await store.exec('CREATE UNIQUE INDEX IF NOT EXISTS name ON GroupKeys (id)') + this.store = store } catch (err) { if (!this.error) { this.error = err @@ -45,39 +52,41 @@ class ServerStorage { async get(key: string) { await this.init() - const value = await this.store.get(key).catch((err) => { - if (err.notFound) { return } - throw err - }) - return value + const value = await this.store!.get('SELECT groupKey FROM GroupKeys WHERE id = ? AND streamId = ?', key, this.streamId) + return value?.groupKey } - async set(key: string, value: any) { + async has(key: string) { await this.init() - return this.store.put(key, value) + const value = await this.store!.get('SELECT COUNT(*) FROM GroupKeys WHERE id = ? AND streamId = ?', key, this.streamId) + return value && value['COUNT(*)'] != null && value['COUNT(*)'] !== 0 } - async delete(key: string) { + async set(key: string, value: string) { await this.init() - return this.store.del(key).catch((err) => { - if (err.notFound) { return } - throw err + return this.store!.run('INSERT INTO GroupKeys VALUES ($id, $groupKey, $streamId) ON CONFLICT DO NOTHING', { + $id: key, + $groupKey: value, + $streamId: this.streamId, }) } + async delete(key: string) { + await this.init() + const result = await this.store!.run('DELETE FROM GroupKeys WHERE id = ? AND streamId = ?', key, this.streamId) + return !!result?.changes + } + async clear() { await this.init() - return this.store.clear() + const result = await this.store!.run('DELETE FROM GroupKeys WHERE streamId = ?', this.streamId) + return !!result?.changes } async size() { await this.init() - let count = 0 - const keyStream = this.store.createKeyStream({ keys: false, values: true }).on('data', () => { - count += 1 - }) - await once(keyStream, 'end') - return count + const size = await this.store!.get('SELECT COUNT(*) FROM GroupKeys WHERE streamId = ?;', this.streamId) + return size && size['COUNT(*)'] } get [Symbol.toStringTag]() { @@ -88,12 +97,11 @@ class ServerStorage { export default class GroupKeyStore { store: ServerStorage constructor({ clientId, streamId }: { clientId: string, streamId: string }) { - this.store = new ServerStorage(`${clientId}-${streamId}`) + this.store = new ServerStorage({ clientId, streamId }) } async has(groupKeyId: string) { - const value = await this.store.get(groupKeyId) - return value != null + return this.store.has(groupKeyId) } async size() { @@ -103,11 +111,16 @@ export default class GroupKeyStore { async get(groupKeyId: string) { const value = await this.store.get(groupKeyId) if (!value) { return undefined } - return GroupKey.from(value) + return GroupKey.from([groupKeyId, value]) + } + + async add(groupKey: GroupKey) { + return this.set(groupKey.id, groupKey) } async set(groupKeyId: string, value: GroupKey) { - return this.store.set(groupKeyId, value) + GroupKey.validate(value) + return this.store.set(groupKeyId, value.hex) } async delete(groupKeyId: string) { diff --git a/test/unit/GroupKeyStore.test.ts b/test/unit/GroupKeyStore.test.ts new file mode 100644 index 000000000..924b40b74 --- /dev/null +++ b/test/unit/GroupKeyStore.test.ts @@ -0,0 +1,147 @@ +import crypto from 'crypto' +import { GroupKey } from '../../src/stream/Encryption' +import GroupKeyStore from '../../src/stream/PersistentStore' +import { uid, addAfterFn } from '../utils' + +describe('GroupKeyStore', () => { + let clientId: string + let streamId: string + let store: GroupKeyStore + + const addAfter = addAfterFn() + + beforeEach(() => { + clientId = `0x${crypto.randomBytes(20).toString('hex')}` + streamId = uid('stream') + store = new GroupKeyStore({ + clientId, + streamId, + }) + }) + + afterEach(async () => { + if (!store) { return } + await store.clear() + }) + + it('can get set and delete', async () => { + const groupKey = GroupKey.generate() + expect(await store.has(groupKey.id)).toBeFalsy() + expect(await store.size()).toBe(0) + expect(await store.get(groupKey.id)).toBeFalsy() + expect(await store.delete(groupKey.id)).toBeFalsy() + expect(await store.clear()).toBeFalsy() + + expect(await store.add(groupKey)).toBeTruthy() + expect(await store.add(groupKey)).toBeFalsy() + expect(await store.has(groupKey.id)).toBeTruthy() + expect(await store.get(groupKey.id)).toEqual(groupKey) + expect(await store.size()).toBe(1) + expect(await store.delete(groupKey.id)).toBeTruthy() + + expect(await store.has(groupKey.id)).toBeFalsy() + expect(await store.size()).toBe(0) + + expect(await store.get(groupKey.id)).toBeFalsy() + expect(await store.delete(groupKey.id)).toBeFalsy() + expect(await store.add(groupKey)).toBeTruthy() + expect(await store.size()).toBe(1) + expect(await store.clear()).toBeTruthy() + expect(await store.size()).toBe(0) + }) + + it('can get set and delete in parallel', async () => { + const store2 = new GroupKeyStore({ + clientId, + streamId, + }) + addAfter(() => store2.clear()) + + for (let i = 0; i < 5; i++) { + const groupKey = GroupKey.generate() + /* eslint-disable no-await-in-loop, no-loop-func, promise/always-return */ + const tasks = [ + // test adding to same store in parallel doesn't break + // add key to store1 twice in parallel + store.add(groupKey).then(async () => { + // immediately check exists in store2 + expect(await store2.has(groupKey.id)).toBeTruthy() + }), + store.add(groupKey).then(async () => { + // immediately check exists in store2 + expect(await store2.has(groupKey.id)).toBeTruthy() + }), + // test adding to another store at same time doesn't break + // add to store2 in parallel + store2.add(groupKey).then(async () => { + // immediately check exists in store1 + expect(await store.has(groupKey.id)).toBeTruthy() + }), + ] + + await Promise.allSettled(tasks) + await Promise.all(tasks) + /* eslint-enable no-await-in-loop, no-loop-func, promise/always-return */ + } + }) + + it('does not conflict with other streamIds', async () => { + const streamId2 = uid('stream') + const store2 = new GroupKeyStore({ + clientId, + streamId: streamId2, + }) + + addAfter(() => store2.clear()) + + const groupKey = GroupKey.generate() + await store.add(groupKey) + expect(await store2.has(groupKey.id)).toBeFalsy() + expect(await store2.get(groupKey.id)).toBeFalsy() + expect(await store2.size()).toBe(0) + expect(await store2.delete(groupKey.id)).toBeFalsy() + expect(await store2.clear()).toBeFalsy() + expect(await store2.clear()).toBeFalsy() + expect(await store.get(groupKey.id)).toEqual(groupKey) + }) + + it('does not conflict with other clientIds', async () => { + const clientId2 = `0x${crypto.randomBytes(20).toString('hex')}` + const store2 = new GroupKeyStore({ + clientId: clientId2, + streamId, + }) + + addAfter(() => store2.clear()) + + const groupKey = GroupKey.generate() + await store.add(groupKey) + expect(await store2.has(groupKey.id)).toBeFalsy() + expect(await store2.get(groupKey.id)).toBeFalsy() + expect(await store2.size()).toBe(0) + expect(await store2.delete(groupKey.id)).toBeFalsy() + expect(await store2.clear()).toBeFalsy() + expect(await store2.clear()).toBeFalsy() + expect(await store.get(groupKey.id)).toEqual(groupKey) + }) + + it('does not conflict with other clientIds', async () => { + const clientId2 = `0x${crypto.randomBytes(20).toString('hex')}` + const store2 = new GroupKeyStore({ + clientId: clientId2, + streamId, + }) + + addAfter(() => store2.clear()) + + const groupKey = GroupKey.generate() + await store.add(groupKey) + expect(await store2.has(groupKey.id)).toBeFalsy() + expect(await store2.get(groupKey.id)).toBeFalsy() + expect(await store2.size()).toBe(0) + expect(await store2.delete(groupKey.id)).toBeFalsy() + expect(await store2.clear()).toBeFalsy() + expect(await store2.clear()).toBeFalsy() + expect(await store.get(groupKey.id)).toEqual(groupKey) + }) +}) From fbb62bfdcc333d22fe9b9e236055380636333207 Mon Sep 17 00:00:00 2001 From: Tim Oxley Date: Wed, 5 May 2021 16:38:51 -0400 Subject: [PATCH 17/28] fix: Improve group key store. --- src/stream/KeyExchange.ts | 186 +++++++++++++++------------- src/stream/PersistentStore.ts | 4 +- test/integration/Encryption.test.js | 2 +- test/unit/GroupKeyStore.test.ts | 69 +++++++++-- 4 files changed, 162 insertions(+), 99 deletions(-) diff --git a/src/stream/KeyExchange.ts b/src/stream/KeyExchange.ts index d7115f034..fcc4ff5db 100644 --- a/src/stream/KeyExchange.ts +++ b/src/stream/KeyExchange.ts @@ -70,24 +70,27 @@ type GroupKeyStoreOptions = { groupKeys: [GroupKeyId, GroupKey][] } -function GroupKeyStore({ clientId, streamId, groupKeys }: GroupKeyStoreOptions) { - const store = new PersistentStore({ clientId, streamId }) - - let currentGroupKeyId: GroupKeyId | undefined // current key id if any - const nextGroupKeys: GroupKey[] = [] // the keys to use next, disappears if not actually used. Max queue size 2 - - groupKeys.forEach(([groupKeyId, groupKey]) => { - GroupKey.validate(groupKey) - if (groupKeyId !== groupKey.id) { - throw new Error(`Ids must match: groupKey.id: ${groupKey.id}, groupKeyId: ${groupKeyId}`) - } - // use last init key as current - currentGroupKeyId = groupKey.id - }) +export class GroupKeyStore { + store + currentGroupKeyId: GroupKeyId | undefined // current key id if any + nextGroupKeys: GroupKey[] = [] // the keys to use next, disappears if not actually used. Max queue size 2 + + constructor({ clientId, streamId, groupKeys }: GroupKeyStoreOptions) { + this.store = new PersistentStore({ clientId, streamId }) + + groupKeys.forEach(([groupKeyId, groupKey]) => { + GroupKey.validate(groupKey) + if (groupKeyId !== groupKey.id) { + throw new Error(`Ids must match: groupKey.id: ${groupKey.id}, groupKeyId: ${groupKeyId}`) + } + // use last init key as current + this.currentGroupKeyId = groupKey.id + }) + } - async function storeKey(groupKey: GroupKey) { + private async storeKey(groupKey: GroupKey) { GroupKey.validate(groupKey) - const existingKey = await store.get(groupKey.id) + const existingKey = await this.store.get(groupKey.id) if (existingKey) { if (!existingKey.equals(groupKey)) { throw new GroupKey.InvalidGroupKeyError( @@ -96,90 +99,95 @@ function GroupKeyStore({ clientId, streamId, groupKeys }: GroupKeyStoreOptions) ) } - await store.set(groupKey.id, existingKey) + await this.store.set(groupKey.id, existingKey) return existingKey } - await store.set(groupKey.id, groupKey) + await this.store.set(groupKey.id, groupKey) return groupKey } - return { - async has(id: GroupKeyId) { - if (currentGroupKeyId === id) { return true } + async has(id: GroupKeyId) { + if (this.currentGroupKeyId === id) { return true } - if (nextGroupKeys.some((nextKey) => nextKey.id === id)) { return true } + if (this.nextGroupKeys.some((nextKey) => nextKey.id === id)) { return true } - return store.has(id) - }, - async isEmpty() { - return !nextGroupKeys.length && await store.size() === 0 - }, - async useGroupKey(): Promise<[GroupKey | undefined, GroupKey | undefined]> { - const nextGroupKey = nextGroupKeys.pop() - // First use of group key on this stream, no current key. Make next key current. - if (!currentGroupKeyId && nextGroupKey) { - await storeKey(nextGroupKey) - currentGroupKeyId = nextGroupKey.id - return [ - await this.get(currentGroupKeyId), - undefined, - ] - } + return this.store.has(id) + } - // Keep using current key (empty next) - if (currentGroupKeyId != null && !nextGroupKey) { - return [ - await this.get(currentGroupKeyId), - undefined - ] - } + async isEmpty() { + return !this.nextGroupKeys.length && await this.store.size() === 0 + } - // Key changed (non-empty next). return current + next. Make next key current. - if (currentGroupKeyId != null && nextGroupKey != null) { - await storeKey(nextGroupKey) - const prevGroupKey = await this.get(currentGroupKeyId) - currentGroupKeyId = nextGroupKey.id - // use current key one more time - return [ - prevGroupKey, - nextGroupKey, - ] - } + async useGroupKey(): Promise<[GroupKey | undefined, GroupKey | undefined]> { + const nextGroupKey = this.nextGroupKeys.pop() + // First use of group key on this stream, no current key. Make next key current. + if (!this.currentGroupKeyId && nextGroupKey) { + this.currentGroupKeyId = nextGroupKey.id + return [ + await this.get(this.currentGroupKeyId), + undefined, + ] + } - // Generate & use new key if none already set. - await this.rotateGroupKey() - return this.useGroupKey() - }, - async get(id: GroupKeyId) { - return store.get(id) - }, - async clear() { - currentGroupKeyId = undefined - nextGroupKeys.length = 0 - return store.clear() - }, - async rotateGroupKey() { - return this.setNextGroupKey(GroupKey.generate()) - }, - async add(groupKey: GroupKey) { - return storeKey(groupKey) - }, - async setNextGroupKey(newKey: GroupKey) { - GroupKey.validate(newKey) - nextGroupKeys.unshift(newKey) - nextGroupKeys.length = Math.min(nextGroupKeys.length, 2) - }, - async rekey() { - const newKey = GroupKey.generate() - await storeKey(newKey) - currentGroupKeyId = newKey.id - nextGroupKeys.length = 0 + // Keep using current key (empty next) + if (this.currentGroupKeyId != null && !nextGroupKey) { + return [ + await this.get(this.currentGroupKeyId), + undefined + ] + } + + // Key changed (non-empty next). return current + next. Make next key current. + if (this.currentGroupKeyId != null && nextGroupKey != null) { + const prevId = this.currentGroupKeyId + this.currentGroupKeyId = nextGroupKey.id + const prevGroupKey = await this.get(prevId) + // use current key one more time + return [ + prevGroupKey, + nextGroupKey, + ] } + + // Generate & use new key if none already set. + await this.rotateGroupKey() + return this.useGroupKey() + } + + async get(id: GroupKeyId) { + return this.store.get(id) + } + + async clear() { + this.currentGroupKeyId = undefined + this.nextGroupKeys.length = 0 + return this.store.clear() + } + + async rotateGroupKey() { + return this.setNextGroupKey(GroupKey.generate()) + } + + async add(groupKey: GroupKey) { + return this.storeKey(groupKey) + } + + async setNextGroupKey(newKey: GroupKey) { + GroupKey.validate(newKey) + this.nextGroupKeys.unshift(newKey) + this.nextGroupKeys.length = Math.min(this.nextGroupKeys.length, 2) + await this.storeKey(newKey) + } + + async rekey() { + const newKey = GroupKey.generate() + await this.storeKey(newKey) + this.currentGroupKeyId = newKey.id + this.nextGroupKeys.length = 0 } } -type GroupKeyStorage = ReturnType type GroupKeysSerialized = Record function parseGroupKeys(groupKeys: GroupKeysSerialized = {}): Map { @@ -243,7 +251,7 @@ async function catchKeyExchangeError(client: StreamrClient, streamMessage: Strea } } -async function PublisherKeyExhangeSubscription(client: StreamrClient, getGroupKeyStore: (streamId: string) => Promise) { +async function PublisherKeyExhangeSubscription(client: StreamrClient, getGroupKeyStore: (streamId: string) => Promise) { async function onKeyExchangeMessage(_parsedContent: any, streamMessage: StreamMessage) { return catchKeyExchangeError(client, streamMessage, async () => { if (streamMessage.messageType !== StreamMessage.MESSAGE_TYPES.GROUP_KEY_REQUEST) { @@ -316,7 +324,7 @@ export function PublisherKeyExhange(client: StreamrClient, { groupKeys = {} }: K let enabled = true const getGroupKeyStore = pMemoize(async (streamId) => { const clientId = await client.getAddress() - return GroupKeyStore({ + return new GroupKeyStore({ clientId, streamId, groupKeys: [...parseGroupKeys(groupKeys[streamId]).entries()] @@ -403,7 +411,7 @@ async function getGroupKeysFromStreamMessage(streamMessage: StreamMessage, encry async function SubscriberKeyExhangeSubscription( client: StreamrClient, - getGroupKeyStore: (streamId: string) => Promise, + getGroupKeyStore: (streamId: string) => Promise, encryptionUtil: EncryptionUtil ) { let sub: Subscription @@ -436,7 +444,7 @@ export function SubscriberKeyExchange(client: StreamrClient, { groupKeys = {} }: const getGroupKeyStore = pMemoize(async (streamId) => { const clientId = await client.getAddress() - return GroupKeyStore({ + return new GroupKeyStore({ clientId, streamId, groupKeys: [...parseGroupKeys(groupKeys[streamId]).entries()] diff --git a/src/stream/PersistentStore.ts b/src/stream/PersistentStore.ts index 935b76980..ad41ef7b2 100644 --- a/src/stream/PersistentStore.ts +++ b/src/stream/PersistentStore.ts @@ -64,11 +64,13 @@ class ServerStorage { async set(key: string, value: string) { await this.init() - return this.store!.run('INSERT INTO GroupKeys VALUES ($id, $groupKey, $streamId) ON CONFLICT DO NOTHING', { + const result = await this.store!.run('INSERT INTO GroupKeys VALUES ($id, $groupKey, $streamId) ON CONFLICT DO NOTHING', { $id: key, $groupKey: value, $streamId: this.streamId, }) + + return !!result?.changes } async delete(key: string) { diff --git a/test/integration/Encryption.test.js b/test/integration/Encryption.test.js index 8822a9925..ce817d17f 100644 --- a/test/integration/Encryption.test.js +++ b/test/integration/Encryption.test.js @@ -140,7 +140,7 @@ describeRepeats('decryption', () => { })) client.once('error', done.reject) - client.setNextGroupKey(stream.id, groupKey) + await client.setNextGroupKey(stream.id, groupKey) // Publish after subscribed await Promise.all([ client.publish(stream.id, msg), diff --git a/test/unit/GroupKeyStore.test.ts b/test/unit/GroupKeyStore.test.ts index 924b40b74..46dd8475d 100644 --- a/test/unit/GroupKeyStore.test.ts +++ b/test/unit/GroupKeyStore.test.ts @@ -1,19 +1,72 @@ import crypto from 'crypto' import { GroupKey } from '../../src/stream/Encryption' -import GroupKeyStore from '../../src/stream/PersistentStore' -import { uid, addAfterFn } from '../utils' +import PersistentStore from '../../src/stream/PersistentStore' +import { GroupKeyStore } from '../../src/stream/KeyExchange' +import { uid, addAfterFn, describeRepeats } from '../utils' -describe('GroupKeyStore', () => { +describeRepeats('GroupKeyStore', () => { let clientId: string let streamId: string let store: GroupKeyStore + beforeEach(() => { + clientId = `0x${crypto.randomBytes(20).toString('hex')}` + streamId = uid('stream') + store = new GroupKeyStore({ + clientId, + streamId, + groupKeys: [], + }) + }) + + afterEach(async () => { + if (!store) { return } + await store.clear() + }) + + it('can get set and delete', async () => { + const groupKey = GroupKey.generate() + expect(await store.get(groupKey.id)).toBeFalsy() + expect(await store.add(groupKey)).toBeTruthy() + expect(await store.get(groupKey.id)).toEqual(groupKey) + expect(await store.clear()).toBeTruthy() + expect(await store.clear()).toBeFalsy() + expect(await store.get(groupKey.id)).toBeFalsy() + }) + + it('can set next and use', async () => { + const groupKey = GroupKey.generate() + await store.setNextGroupKey(groupKey) + expect(await store.useGroupKey()).toEqual([groupKey, undefined]) + expect(await store.useGroupKey()).toEqual([groupKey, undefined]) + const groupKey2 = GroupKey.generate() + await store.setNextGroupKey(groupKey2) + expect(await store.useGroupKey()).toEqual([groupKey, groupKey2]) + expect(await store.useGroupKey()).toEqual([groupKey2, undefined]) + }) + + it('can set next in parallel and use', async () => { + const groupKey = GroupKey.generate() + const groupKey2 = GroupKey.generate() + await Promise.all([ + store.setNextGroupKey(groupKey), + store.setNextGroupKey(groupKey2), + ]) + expect(await store.useGroupKey()).toEqual([groupKey, undefined]) + }) +}) + +describeRepeats('PersistentStore', () => { + let clientId: string + let streamId: string + let store: PersistentStore + const addAfter = addAfterFn() beforeEach(() => { clientId = `0x${crypto.randomBytes(20).toString('hex')}` streamId = uid('stream') - store = new GroupKeyStore({ + store = new PersistentStore({ clientId, streamId, }) @@ -51,7 +104,7 @@ describe('GroupKeyStore', () => { }) it('can get set and delete in parallel', async () => { - const store2 = new GroupKeyStore({ + const store2 = new PersistentStore({ clientId, streamId, }) @@ -87,7 +140,7 @@ describe('GroupKeyStore', () => { it('does not conflict with other streamIds', async () => { const streamId2 = uid('stream') - const store2 = new GroupKeyStore({ + const store2 = new PersistentStore({ clientId, streamId: streamId2, }) @@ -107,7 +160,7 @@ describe('GroupKeyStore', () => { it('does not conflict with other clientIds', async () => { const clientId2 = `0x${crypto.randomBytes(20).toString('hex')}` - const store2 = new GroupKeyStore({ + const store2 = new PersistentStore({ clientId: clientId2, streamId, }) @@ -127,7 +180,7 @@ describe('GroupKeyStore', () => { it('does not conflict with other clientIds', async () => { const clientId2 = `0x${crypto.randomBytes(20).toString('hex')}` - const store2 = new GroupKeyStore({ + const store2 = new PersistentStore({ clientId: clientId2, streamId, }) From 378ef96c2b0f3b841fdf36e3e152628432be5da7 Mon Sep 17 00:00:00 2001 From: Tim Oxley Date: Thu, 6 May 2021 09:17:18 -0400 Subject: [PATCH 18/28] fix: fs/promises not supported on node 12. --- src/stream/PersistentStore.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/stream/PersistentStore.ts b/src/stream/PersistentStore.ts index ad41ef7b2..9dd563ba1 100644 --- a/src/stream/PersistentStore.ts +++ b/src/stream/PersistentStore.ts @@ -1,6 +1,6 @@ import envPaths from 'env-paths' import { dirname, join } from 'path' -import fs from 'fs/promises' +import { promises as fs } from 'fs' import { open, Database } from 'sqlite' import sqlite3 from 'sqlite3' From 49c44cbb9124345b9089a1dbf1473dac81c02b86 Mon Sep 17 00:00:00 2001 From: Tim Oxley Date: Thu, 6 May 2021 14:16:07 -0400 Subject: [PATCH 19/28] refactor: Move encryption and key exchange files under stream/encryption. --- src/StreamrClient.ts | 3 +- src/index.ts | 2 +- src/publish/Encrypt.ts | 4 +- src/rest/StreamEndpoints.ts | 2 +- src/stream/PersistentStore.ts | 139 --------- src/stream/{ => encryption}/BrowserStore.ts | 0 src/stream/{ => encryption}/Encryption.ts | 2 +- src/stream/encryption/GroupKeyStore.ts | 274 ++++++++++++++++++ src/stream/{ => encryption}/KeyExchange.ts | 137 +-------- src/stream/encryption/ServerStore.ts | 0 src/stream/index.ts | 2 + src/subscribe/Decrypt.js | 4 +- test/integration/Encryption.test.js | 2 +- .../EncryptionKeyPersistence.test.ts | 2 +- test/unit/Encryption.test.ts | 2 +- test/unit/GroupKeyStore.test.ts | 17 +- 16 files changed, 302 insertions(+), 290 deletions(-) delete mode 100644 src/stream/PersistentStore.ts rename src/stream/{ => encryption}/BrowserStore.ts (100%) rename src/stream/{ => encryption}/Encryption.ts (99%) create mode 100644 src/stream/encryption/GroupKeyStore.ts rename src/stream/{ => encryption}/KeyExchange.ts (81%) create mode 100644 src/stream/encryption/ServerStore.ts diff --git a/src/StreamrClient.ts b/src/StreamrClient.ts index 365da657f..9a60cdb09 100644 --- a/src/StreamrClient.ts +++ b/src/StreamrClient.ts @@ -22,8 +22,7 @@ import { DataUnion, DataUnionDeployOptions } from './dataunion/DataUnion' import { BigNumber } from '@ethersproject/bignumber' import { getAddress } from '@ethersproject/address' import { Contract } from '@ethersproject/contracts' -import { StreamPartDefinition } from './stream' -import type { GroupKey } from './stream/Encryption' +import { StreamPartDefinition, GroupKey } from './stream' // TODO get metadata type from streamr-protocol-js project (it doesn't export the type definitions yet) export type OnMessageCallback = MaybeAsync<(message: any, metadata: any) => void> diff --git a/src/index.ts b/src/index.ts index 82cd3e739..06893b57f 100644 --- a/src/index.ts +++ b/src/index.ts @@ -10,7 +10,7 @@ import { StreamrClient } from './StreamrClient' export * from './StreamrClient' export * from './Config' export * from './stream' -export * from './stream/Encryption' +export * from './stream/encryption/Encryption' export * from './stream/StreamPart' export * from './stream/StorageNode' export * from './subscribe' diff --git a/src/publish/Encrypt.ts b/src/publish/Encrypt.ts index 839c48e3a..a191a2231 100644 --- a/src/publish/Encrypt.ts +++ b/src/publish/Encrypt.ts @@ -1,9 +1,9 @@ import { MessageLayer } from 'streamr-client-protocol' -import EncryptionUtil from '../stream/Encryption' +import EncryptionUtil from '../stream/encryption/Encryption' import { Stream } from '../stream' import { StreamrClient } from '../StreamrClient' -import { PublisherKeyExhange } from '../stream/KeyExchange' +import { PublisherKeyExhange } from '../stream/encryption/KeyExchange' const { StreamMessage } = MessageLayer diff --git a/src/rest/StreamEndpoints.ts b/src/rest/StreamEndpoints.ts index ca237af00..51e0f9b9c 100644 --- a/src/rest/StreamEndpoints.ts +++ b/src/rest/StreamEndpoints.ts @@ -8,7 +8,7 @@ import { getEndpointUrl } from '../utils' import { validateOptions } from '../stream/utils' import { Stream, StreamOperation, StreamProperties } from '../stream' import { StreamPart } from '../stream/StreamPart' -import { isKeyExchangeStream } from '../stream/KeyExchange' +import { isKeyExchangeStream } from '../stream/encryption/KeyExchange' import authFetch, { ErrorCode, NotFoundError } from './authFetch' import { EthereumAddress } from '../types' diff --git a/src/stream/PersistentStore.ts b/src/stream/PersistentStore.ts deleted file mode 100644 index 9dd563ba1..000000000 --- a/src/stream/PersistentStore.ts +++ /dev/null @@ -1,139 +0,0 @@ -import envPaths from 'env-paths' -import { dirname, join } from 'path' -import { promises as fs } from 'fs' -import { open, Database } from 'sqlite' -import sqlite3 from 'sqlite3' - -import { GroupKey } from './Encryption' -import { pOnce } from '../utils' - -class ServerStorage { - readonly clientId: string - readonly streamId: string - readonly dbFilePath: string - private store?: Database - private error?: Error - - constructor({ clientId, streamId }: { clientId: string, streamId: string }) { - this.streamId = encodeURIComponent(streamId) - this.clientId = encodeURIComponent(clientId) - const paths = envPaths('streamr-client') - const dbFilePath = join(paths.data, clientId, 'GroupKeys.db') - this.dbFilePath = dbFilePath - - this.init = pOnce(this.init.bind(this)) - } - - async init() { - try { - await fs.mkdir(dirname(this.dbFilePath), { recursive: true }) - // open the database - const store = await open({ - filename: this.dbFilePath, - driver: sqlite3.Database - }) - await store.exec(`CREATE TABLE IF NOT EXISTS GroupKeys ( - id TEXT, - groupKey TEXT, - streamId TEXT - )`) - await store.exec('CREATE UNIQUE INDEX IF NOT EXISTS name ON GroupKeys (id)') - this.store = store - } catch (err) { - if (!this.error) { - this.error = err - } - } - - if (this.error) { - throw this.error - } - } - - async get(key: string) { - await this.init() - const value = await this.store!.get('SELECT groupKey FROM GroupKeys WHERE id = ? AND streamId = ?', key, this.streamId) - return value?.groupKey - } - - async has(key: string) { - await this.init() - const value = await this.store!.get('SELECT COUNT(*) FROM GroupKeys WHERE id = ? AND streamId = ?', key, this.streamId) - return value && value['COUNT(*)'] != null && value['COUNT(*)'] !== 0 - } - - async set(key: string, value: string) { - await this.init() - const result = await this.store!.run('INSERT INTO GroupKeys VALUES ($id, $groupKey, $streamId) ON CONFLICT DO NOTHING', { - $id: key, - $groupKey: value, - $streamId: this.streamId, - }) - - return !!result?.changes - } - - async delete(key: string) { - await this.init() - const result = await this.store!.run('DELETE FROM GroupKeys WHERE id = ? AND streamId = ?', key, this.streamId) - return !!result?.changes - } - - async clear() { - await this.init() - const result = await this.store!.run('DELETE FROM GroupKeys WHERE streamId = ?', this.streamId) - return !!result?.changes - } - - async size() { - await this.init() - const size = await this.store!.get('SELECT COUNT(*) FROM GroupKeys WHERE streamId = ?;', this.streamId) - return size && size['COUNT(*)'] - } - - get [Symbol.toStringTag]() { - return this.constructor.name - } -} - -export default class GroupKeyStore { - store: ServerStorage - constructor({ clientId, streamId }: { clientId: string, streamId: string }) { - this.store = new ServerStorage({ clientId, streamId }) - } - - async has(groupKeyId: string) { - return this.store.has(groupKeyId) - } - - async size() { - return this.store.size() - } - - async get(groupKeyId: string) { - const value = await this.store.get(groupKeyId) - if (!value) { return undefined } - return GroupKey.from([groupKeyId, value]) - } - - async add(groupKey: GroupKey) { - return this.set(groupKey.id, groupKey) - } - - async set(groupKeyId: string, value: GroupKey) { - GroupKey.validate(value) - return this.store.set(groupKeyId, value.hex) - } - - async delete(groupKeyId: string) { - return this.store.delete(groupKeyId) - } - - async clear() { - return this.store.clear() - } - - get [Symbol.toStringTag]() { - return this.constructor.name - } -} diff --git a/src/stream/BrowserStore.ts b/src/stream/encryption/BrowserStore.ts similarity index 100% rename from src/stream/BrowserStore.ts rename to src/stream/encryption/BrowserStore.ts diff --git a/src/stream/Encryption.ts b/src/stream/encryption/Encryption.ts similarity index 99% rename from src/stream/Encryption.ts rename to src/stream/encryption/Encryption.ts index ea7bdbae3..1ef44fb27 100644 --- a/src/stream/Encryption.ts +++ b/src/stream/encryption/Encryption.ts @@ -7,7 +7,7 @@ import { Crypto } from 'node-webcrypto-ossl' import { arrayify, hexlify } from '@ethersproject/bytes' import { MessageLayer } from 'streamr-client-protocol' -import { uuid } from '../utils' +import { uuid } from '../../utils' const { StreamMessage, EncryptedGroupKey } = MessageLayer diff --git a/src/stream/encryption/GroupKeyStore.ts b/src/stream/encryption/GroupKeyStore.ts new file mode 100644 index 000000000..ac253d380 --- /dev/null +++ b/src/stream/encryption/GroupKeyStore.ts @@ -0,0 +1,274 @@ +import envPaths from 'env-paths' +import { dirname, join } from 'path' +import { promises as fs } from 'fs' +import { open, Database } from 'sqlite' +import sqlite3 from 'sqlite3' + +import { GroupKey } from './Encryption' +import { pOnce } from '../../utils' + +interface Storage { + get(key: K): Promise + set(key: K, value: V): Promise + has(key: K): Promise + delete(key: K): Promise + clear(): Promise + size(): Promise +} + +class ServerStorage implements Storage { + readonly clientId: string + readonly streamId: string + readonly dbFilePath: string + private store?: Database + private error?: Error + + constructor({ clientId, streamId }: { clientId: string, streamId: string }) { + this.streamId = encodeURIComponent(streamId) + this.clientId = encodeURIComponent(clientId) + const paths = envPaths('streamr-client') + const dbFilePath = join(paths.data, clientId, 'GroupKeys.db') + this.dbFilePath = dbFilePath + + this.init = pOnce(this.init.bind(this)) + } + + async init() { + try { + await fs.mkdir(dirname(this.dbFilePath), { recursive: true }) + // open the database + const store = await open({ + filename: this.dbFilePath, + driver: sqlite3.Database + }) + await store.exec(`CREATE TABLE IF NOT EXISTS GroupKeys ( + id TEXT, + groupKey TEXT, + streamId TEXT + )`) + await store.exec('CREATE UNIQUE INDEX IF NOT EXISTS name ON GroupKeys (id)') + this.store = store + } catch (err) { + if (!this.error) { + this.error = err + } + } + + if (this.error) { + throw this.error + } + } + + async get(key: string) { + await this.init() + const value = await this.store!.get('SELECT groupKey FROM GroupKeys WHERE id = ? AND streamId = ?', key, this.streamId) + return value?.groupKey + } + + async has(key: string) { + await this.init() + const value = await this.store!.get('SELECT COUNT(*) FROM GroupKeys WHERE id = ? AND streamId = ?', key, this.streamId) + return value && value['COUNT(*)'] != null && value['COUNT(*)'] !== 0 + } + + async set(key: string, value: string) { + await this.init() + const result = await this.store!.run('INSERT INTO GroupKeys VALUES ($id, $groupKey, $streamId) ON CONFLICT DO NOTHING', { + $id: key, + $groupKey: value, + $streamId: this.streamId, + }) + + return !!result?.changes + } + + async delete(key: string) { + await this.init() + const result = await this.store!.run('DELETE FROM GroupKeys WHERE id = ? AND streamId = ?', key, this.streamId) + return !!result?.changes + } + + async clear() { + await this.init() + const result = await this.store!.run('DELETE FROM GroupKeys WHERE streamId = ?', this.streamId) + return !!result?.changes + } + + async size() { + await this.init() + const size = await this.store!.get('SELECT COUNT(*) FROM GroupKeys WHERE streamId = ?;', this.streamId) + return size && size['COUNT(*)'] + } + + get [Symbol.toStringTag]() { + return this.constructor.name + } +} + +type GroupKeyId = string + +type GroupKeyStoreOptions = { + clientId: string, + streamId: string, + groupKeys: [GroupKeyId, GroupKey][] +} + +export class GroupKeyPersistence { + store: Storage + constructor({ clientId, streamId }: { clientId: string, streamId: string }) { + this.store = new ServerStorage({ clientId, streamId }) + } + + async has(groupKeyId: string) { + return this.store.has(groupKeyId) + } + + async size() { + return this.store.size() + } + + async get(groupKeyId: string) { + const value = await this.store.get(groupKeyId) + if (!value) { return undefined } + return GroupKey.from([groupKeyId, value]) + } + + async add(groupKey: GroupKey) { + return this.set(groupKey.id, groupKey) + } + + async set(groupKeyId: string, value: GroupKey) { + GroupKey.validate(value) + return this.store.set(groupKeyId, value.hex) + } + + async delete(groupKeyId: string) { + return this.store.delete(groupKeyId) + } + + async clear() { + return this.store.clear() + } + + get [Symbol.toStringTag]() { + return this.constructor.name + } +} + +export default class GroupKeyStore { + store + currentGroupKeyId: GroupKeyId | undefined // current key id if any + nextGroupKeys: GroupKey[] = [] // the keys to use next, disappears if not actually used. Max queue size 2 + + constructor({ clientId, streamId, groupKeys }: GroupKeyStoreOptions) { + this.store = new GroupKeyPersistence({ clientId, streamId }) + + groupKeys.forEach(([groupKeyId, groupKey]) => { + GroupKey.validate(groupKey) + if (groupKeyId !== groupKey.id) { + throw new Error(`Ids must match: groupKey.id: ${groupKey.id}, groupKeyId: ${groupKeyId}`) + } + // use last init key as current + this.currentGroupKeyId = groupKey.id + }) + } + + private async storeKey(groupKey: GroupKey) { + GroupKey.validate(groupKey) + const existingKey = await this.store.get(groupKey.id) + if (existingKey) { + if (!existingKey.equals(groupKey)) { + throw new GroupKey.InvalidGroupKeyError( + `Trying to add groupKey ${groupKey.id} but key exists & is not equivalent to new GroupKey: ${groupKey}.`, + groupKey + ) + } + + await this.store.set(groupKey.id, existingKey) + return existingKey + } + + await this.store.set(groupKey.id, groupKey) + return groupKey + } + + async has(id: GroupKeyId) { + if (this.currentGroupKeyId === id) { return true } + + if (this.nextGroupKeys.some((nextKey) => nextKey.id === id)) { return true } + + return this.store.has(id) + } + + async isEmpty() { + return !this.nextGroupKeys.length && await this.store.size() === 0 + } + + async useGroupKey(): Promise<[GroupKey | undefined, GroupKey | undefined]> { + const nextGroupKey = this.nextGroupKeys.pop() + // First use of group key on this stream, no current key. Make next key current. + if (!this.currentGroupKeyId && nextGroupKey) { + this.currentGroupKeyId = nextGroupKey.id + return [ + await this.get(this.currentGroupKeyId!), + undefined, + ] + } + + // Keep using current key (empty next) + if (this.currentGroupKeyId != null && !nextGroupKey) { + return [ + await this.get(this.currentGroupKeyId), + undefined + ] + } + + // Key changed (non-empty next). return current + next. Make next key current. + if (this.currentGroupKeyId != null && nextGroupKey != null) { + const prevId = this.currentGroupKeyId + this.currentGroupKeyId = nextGroupKey.id + const prevGroupKey = await this.get(prevId) + // use current key one more time + return [ + prevGroupKey, + nextGroupKey, + ] + } + + // Generate & use new key if none already set. + await this.rotateGroupKey() + return this.useGroupKey() + } + + async get(id: GroupKeyId) { + return this.store.get(id) + } + + async clear() { + this.currentGroupKeyId = undefined + this.nextGroupKeys.length = 0 + return this.store.clear() + } + + async rotateGroupKey() { + return this.setNextGroupKey(GroupKey.generate()) + } + + async add(groupKey: GroupKey) { + return this.storeKey(groupKey) + } + + async setNextGroupKey(newKey: GroupKey) { + GroupKey.validate(newKey) + this.nextGroupKeys.unshift(newKey) + this.nextGroupKeys.length = Math.min(this.nextGroupKeys.length, 2) + await this.storeKey(newKey) + } + + async rekey() { + const newKey = GroupKey.generate() + await this.storeKey(newKey) + this.currentGroupKeyId = newKey.id + this.nextGroupKeys.length = 0 + } +} diff --git a/src/stream/KeyExchange.ts b/src/stream/encryption/KeyExchange.ts similarity index 81% rename from src/stream/KeyExchange.ts rename to src/stream/encryption/KeyExchange.ts index fcc4ff5db..844034b73 100644 --- a/src/stream/KeyExchange.ts +++ b/src/stream/encryption/KeyExchange.ts @@ -4,14 +4,14 @@ import { import mem from 'mem' import pMemoize from 'p-memoize' -import { uuid, Defer } from '../utils' -import Scaffold from '../utils/Scaffold' +import { uuid, Defer } from '../../utils' +import Scaffold from '../../utils/Scaffold' -import { validateOptions } from './utils' +import { validateOptions } from '../utils' import EncryptionUtil, { GroupKey, GroupKeyish, StreamMessageProcessingError } from './Encryption' -import type { Subscription } from '../subscribe' -import { StreamrClient } from '../StreamrClient' -import PersistentStore from './PersistentStore' +import type { Subscription } from '../../subscribe' +import { StreamrClient } from '../../StreamrClient' +import GroupKeyStore from './GroupKeyStore' const KEY_EXCHANGE_STREAM_PREFIX = 'SYSTEM/keyexchange' @@ -64,130 +64,6 @@ function getKeyExchangeStreamId(address: Address) { return `${KEY_EXCHANGE_STREAM_PREFIX}/${address.toLowerCase()}` } -type GroupKeyStoreOptions = { - clientId: string, - streamId: string, - groupKeys: [GroupKeyId, GroupKey][] -} - -export class GroupKeyStore { - store - currentGroupKeyId: GroupKeyId | undefined // current key id if any - nextGroupKeys: GroupKey[] = [] // the keys to use next, disappears if not actually used. Max queue size 2 - - constructor({ clientId, streamId, groupKeys }: GroupKeyStoreOptions) { - this.store = new PersistentStore({ clientId, streamId }) - - groupKeys.forEach(([groupKeyId, groupKey]) => { - GroupKey.validate(groupKey) - if (groupKeyId !== groupKey.id) { - throw new Error(`Ids must match: groupKey.id: ${groupKey.id}, groupKeyId: ${groupKeyId}`) - } - // use last init key as current - this.currentGroupKeyId = groupKey.id - }) - } - - private async storeKey(groupKey: GroupKey) { - GroupKey.validate(groupKey) - const existingKey = await this.store.get(groupKey.id) - if (existingKey) { - if (!existingKey.equals(groupKey)) { - throw new GroupKey.InvalidGroupKeyError( - `Trying to add groupKey ${groupKey.id} but key exists & is not equivalent to new GroupKey: ${groupKey}.`, - groupKey - ) - } - - await this.store.set(groupKey.id, existingKey) - return existingKey - } - - await this.store.set(groupKey.id, groupKey) - return groupKey - } - - async has(id: GroupKeyId) { - if (this.currentGroupKeyId === id) { return true } - - if (this.nextGroupKeys.some((nextKey) => nextKey.id === id)) { return true } - - return this.store.has(id) - } - - async isEmpty() { - return !this.nextGroupKeys.length && await this.store.size() === 0 - } - - async useGroupKey(): Promise<[GroupKey | undefined, GroupKey | undefined]> { - const nextGroupKey = this.nextGroupKeys.pop() - // First use of group key on this stream, no current key. Make next key current. - if (!this.currentGroupKeyId && nextGroupKey) { - this.currentGroupKeyId = nextGroupKey.id - return [ - await this.get(this.currentGroupKeyId), - undefined, - ] - } - - // Keep using current key (empty next) - if (this.currentGroupKeyId != null && !nextGroupKey) { - return [ - await this.get(this.currentGroupKeyId), - undefined - ] - } - - // Key changed (non-empty next). return current + next. Make next key current. - if (this.currentGroupKeyId != null && nextGroupKey != null) { - const prevId = this.currentGroupKeyId - this.currentGroupKeyId = nextGroupKey.id - const prevGroupKey = await this.get(prevId) - // use current key one more time - return [ - prevGroupKey, - nextGroupKey, - ] - } - - // Generate & use new key if none already set. - await this.rotateGroupKey() - return this.useGroupKey() - } - - async get(id: GroupKeyId) { - return this.store.get(id) - } - - async clear() { - this.currentGroupKeyId = undefined - this.nextGroupKeys.length = 0 - return this.store.clear() - } - - async rotateGroupKey() { - return this.setNextGroupKey(GroupKey.generate()) - } - - async add(groupKey: GroupKey) { - return this.storeKey(groupKey) - } - - async setNextGroupKey(newKey: GroupKey) { - GroupKey.validate(newKey) - this.nextGroupKeys.unshift(newKey) - this.nextGroupKeys.length = Math.min(this.nextGroupKeys.length, 2) - await this.storeKey(newKey) - } - - async rekey() { - const newKey = GroupKey.generate() - await this.storeKey(newKey) - this.currentGroupKeyId = newKey.id - this.nextGroupKeys.length = 0 - } -} - type GroupKeysSerialized = Record function parseGroupKeys(groupKeys: GroupKeysSerialized = {}): Map { @@ -663,6 +539,7 @@ export function SubscriberKeyExchange(client: StreamrClient, { groupKeys = {} }: if (!streamMessage.newGroupKey) { return } const streamId = streamMessage.getStreamId() const groupKeyStore = await getGroupKeyStore(streamId) + // newGroupKey has been converted into GroupKey const newGroupKey: unknown = streamMessage.newGroupKey await groupKeyStore.add(newGroupKey as GroupKey) }, diff --git a/src/stream/encryption/ServerStore.ts b/src/stream/encryption/ServerStore.ts new file mode 100644 index 000000000..e69de29bb diff --git a/src/stream/index.ts b/src/stream/index.ts index 397b5dda2..d48507947 100644 --- a/src/stream/index.ts +++ b/src/stream/index.ts @@ -3,6 +3,8 @@ import { getAddress } from '@ethersproject/address' import { getEndpointUrl, until } from '../utils' import authFetch from '../rest/authFetch' +export { GroupKey } from './encryption/Encryption' + import { StorageNode } from './StorageNode' import { StreamrClient } from '../StreamrClient' import { EthereumAddress } from '../types' diff --git a/src/subscribe/Decrypt.js b/src/subscribe/Decrypt.js index a670f4201..2f4d3d296 100644 --- a/src/subscribe/Decrypt.js +++ b/src/subscribe/Decrypt.js @@ -1,7 +1,7 @@ import { MessageLayer } from 'streamr-client-protocol' -import EncryptionUtil, { UnableToDecryptError } from '../stream/Encryption' -import { SubscriberKeyExchange } from '../stream/KeyExchange' +import EncryptionUtil, { UnableToDecryptError } from '../stream/encryption/Encryption' +import { SubscriberKeyExchange } from '../stream/encryption/KeyExchange' const { StreamMessage } = MessageLayer diff --git a/test/integration/Encryption.test.js b/test/integration/Encryption.test.js index ce817d17f..6ad6b83df 100644 --- a/test/integration/Encryption.test.js +++ b/test/integration/Encryption.test.js @@ -4,7 +4,7 @@ import { MessageLayer } from 'streamr-client-protocol' import { describeRepeats, fakePrivateKey, uid, Msg, getPublishTestMessages } from '../utils' import { Defer } from '../../src/utils' import { StreamrClient } from '../../src/StreamrClient' -import { GroupKey } from '../../src/stream/Encryption' +import { GroupKey } from '../../src/stream/encryption/Encryption' import Connection from '../../src/Connection' import { StorageNode } from '../../src/stream/StorageNode' diff --git a/test/integration/EncryptionKeyPersistence.test.ts b/test/integration/EncryptionKeyPersistence.test.ts index c105f4ed8..4f8caf199 100644 --- a/test/integration/EncryptionKeyPersistence.test.ts +++ b/test/integration/EncryptionKeyPersistence.test.ts @@ -3,7 +3,7 @@ import { wait } from 'streamr-test-utils' import { describeRepeats, fakePrivateKey, uid, getPublishTestMessages, addAfterFn } from '../utils' import { StreamrClient } from '../../src/StreamrClient' import { Stream, StreamOperation } from '../../src/stream' -import { GroupKey } from '../../src/stream/Encryption' +import { GroupKey } from '../../src/stream/encryption/Encryption' import Connection from '../../src/Connection' import config from './config' diff --git a/test/unit/Encryption.test.ts b/test/unit/Encryption.test.ts index 0919aedef..cb2eb9419 100644 --- a/test/unit/Encryption.test.ts +++ b/test/unit/Encryption.test.ts @@ -3,7 +3,7 @@ import crypto from 'crypto' import { ethers } from 'ethers' import { MessageLayer } from 'streamr-client-protocol' -import EncryptionUtil, { GroupKey } from '../../src/stream/Encryption' +import EncryptionUtil, { GroupKey } from '../../src/stream/encryption/Encryption' const { StreamMessage, MessageID } = MessageLayer diff --git a/test/unit/GroupKeyStore.test.ts b/test/unit/GroupKeyStore.test.ts index 46dd8475d..571844322 100644 --- a/test/unit/GroupKeyStore.test.ts +++ b/test/unit/GroupKeyStore.test.ts @@ -1,7 +1,6 @@ import crypto from 'crypto' -import { GroupKey } from '../../src/stream/Encryption' -import PersistentStore from '../../src/stream/PersistentStore' -import { GroupKeyStore } from '../../src/stream/KeyExchange' +import { GroupKey } from '../../src/stream/encryption/Encryption' +import GroupKeyStore, { GroupKeyPersistence } from '../../src/stream/encryption/GroupKeyStore' import { uid, addAfterFn, describeRepeats } from '../utils' describeRepeats('GroupKeyStore', () => { @@ -59,14 +58,14 @@ describeRepeats('GroupKeyStore', () => { describeRepeats('PersistentStore', () => { let clientId: string let streamId: string - let store: PersistentStore + let store: GroupKeyPersistence const addAfter = addAfterFn() beforeEach(() => { clientId = `0x${crypto.randomBytes(20).toString('hex')}` streamId = uid('stream') - store = new PersistentStore({ + store = new GroupKeyPersistence({ clientId, streamId, }) @@ -104,7 +103,7 @@ describeRepeats('PersistentStore', () => { }) it('can get set and delete in parallel', async () => { - const store2 = new PersistentStore({ + const store2 = new GroupKeyPersistence({ clientId, streamId, }) @@ -140,7 +139,7 @@ describeRepeats('PersistentStore', () => { it('does not conflict with other streamIds', async () => { const streamId2 = uid('stream') - const store2 = new PersistentStore({ + const store2 = new GroupKeyPersistence({ clientId, streamId: streamId2, }) @@ -160,7 +159,7 @@ describeRepeats('PersistentStore', () => { it('does not conflict with other clientIds', async () => { const clientId2 = `0x${crypto.randomBytes(20).toString('hex')}` - const store2 = new PersistentStore({ + const store2 = new GroupKeyPersistence({ clientId: clientId2, streamId, }) @@ -180,7 +179,7 @@ describeRepeats('PersistentStore', () => { it('does not conflict with other clientIds', async () => { const clientId2 = `0x${crypto.randomBytes(20).toString('hex')}` - const store2 = new PersistentStore({ + const store2 = new GroupKeyPersistence({ clientId: clientId2, streamId, }) From 4f62e4be47d6253c2129ba27ffc3d7e9262ff2d4 Mon Sep 17 00:00:00 2001 From: Tim Oxley Date: Thu, 6 May 2021 14:54:57 -0400 Subject: [PATCH 20/28] fix: Reimplement browser group key persistence on top of idb. --- package-lock.json | 5 + package.json | 1 + .../encryption/BrowserPersistentStore.ts | 53 +++++++++ src/stream/encryption/BrowserStore.ts | 83 -------------- src/stream/encryption/GroupKeyStore.ts | 103 +----------------- .../encryption/ServerPersistentStore.ts | 97 +++++++++++++++++ src/stream/encryption/ServerStore.ts | 0 webpack.config.js | 6 +- 8 files changed, 164 insertions(+), 184 deletions(-) create mode 100644 src/stream/encryption/BrowserPersistentStore.ts delete mode 100644 src/stream/encryption/BrowserStore.ts create mode 100644 src/stream/encryption/ServerPersistentStore.ts delete mode 100644 src/stream/encryption/ServerStore.ts diff --git a/package-lock.json b/package-lock.json index cf7903b69..2eca49edd 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8070,6 +8070,11 @@ "safer-buffer": ">= 2.1.2 < 3" } }, + "idb-keyval": { + "version": "5.0.5", + "resolved": "https://registry.npmjs.org/idb-keyval/-/idb-keyval-5.0.5.tgz", + "integrity": "sha512-cqi65rrjhgPExI9vmSU7VcYEbHCUfIBY+9YUWxyr0PyGizptFgGFnvZQ0w+tqOXk1lUcGCZGVLfabf7QnR2S0g==" + }, "ieee754": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", diff --git a/package.json b/package.json index 2cc026710..a8e2ae48e 100644 --- a/package.json +++ b/package.json @@ -153,6 +153,7 @@ "debug": "^4.3.2", "env-paths": "^2.2.1", "eventemitter3": "^4.0.7", + "idb-keyval": "^5.0.5", "lodash": "^4.17.21", "mem": "^8.1.1", "node-abort-controller": "^1.2.1", diff --git a/src/stream/encryption/BrowserPersistentStore.ts b/src/stream/encryption/BrowserPersistentStore.ts new file mode 100644 index 000000000..9ed888532 --- /dev/null +++ b/src/stream/encryption/BrowserPersistentStore.ts @@ -0,0 +1,53 @@ +import { PersistentStore } from './GroupKeyStore' +import { get, set, del, clear, keys, createStore } from 'idb-keyval' + +export default class BrowserPersistentStore implements PersistentStore { + readonly clientId: string + readonly streamId: string + private store?: any + + constructor({ clientId, streamId }: { clientId: string, streamId: string }) { + this.streamId = encodeURIComponent(streamId) + this.clientId = encodeURIComponent(clientId) + this.store = createStore(`streamr-client::${clientId}::${streamId}`, 'GroupKeys') + } + + async has(key: string) { + const val = await this.get(key) + return val == null + } + + async get(key: string) { + return get(key, this.store) + } + + async set(key: string, value: string) { + const had = await this.has(key) + await set(key, value, this.store) + return had + } + + async delete(key: string) { + if (!await this.has(key)) { + return false + } + + await del(key, this.store) + return true + } + + async clear() { + const size = await this.size() + await clear(this.store) + return !!size + } + + async size() { + const allKeys = await keys(this.store) + return allKeys.length + } + + get [Symbol.toStringTag]() { + return this.constructor.name + } +} diff --git a/src/stream/encryption/BrowserStore.ts b/src/stream/encryption/BrowserStore.ts deleted file mode 100644 index e2db609cc..000000000 --- a/src/stream/encryption/BrowserStore.ts +++ /dev/null @@ -1,83 +0,0 @@ -import type { PersistentStorage } from './PersistentStore' - -export default class BrowserStorage implements PersistentStorage { - id: string - constructor(id: string) { - this.id = `BrowserStorage:${id}` - } - - private getData() { - return JSON.parse(window.localStorage.getItem(this.id) || '{}') || {} - } - - private setData(value: any) { - return window.localStorage.setItem(this.id, JSON.stringify(value)) - } - - private mergeData(value: any) { - const data = this.getData() - return window.localStorage.setItem(this.id, JSON.stringify({ - ...data, - ...value - })) - } - - has(key: string) { - return !!this.getData()[key] - } - - get(key: string) { - return this.getData()[key] - } - - keys() { - return Object.keys(this.getData())[Symbol.iterator]() - } - - values() { - return Object.values(this.getData())[Symbol.iterator]() - } - - entries() { - return Object.entries(this.getData())[Symbol.iterator]() - } - - forEach(...args: Parameters['forEach']>) { - return new Map(Object.entries(this.getData())).forEach(...args) - } - - set(key: string, value: any) { - this.mergeData({ - [key]: value, - }) - return this - } - - delete(key: string) { - if (!this.has(key)) { - return false - } - - const data = this.getData() - delete data[key] - this.setData(data) - return true - } - - clear() { - return this.setData({}) - } - - get size() { - const data = this.getData() - return Object.keys(data).length - } - - [Symbol.iterator]() { - return new Map(Object.entries(this.getData()))[Symbol.iterator]() - } - - get [Symbol.toStringTag]() { - return this.constructor.name - } -} diff --git a/src/stream/encryption/GroupKeyStore.ts b/src/stream/encryption/GroupKeyStore.ts index ac253d380..d9f09e1c2 100644 --- a/src/stream/encryption/GroupKeyStore.ts +++ b/src/stream/encryption/GroupKeyStore.ts @@ -1,13 +1,7 @@ -import envPaths from 'env-paths' -import { dirname, join } from 'path' -import { promises as fs } from 'fs' -import { open, Database } from 'sqlite' -import sqlite3 from 'sqlite3' - import { GroupKey } from './Encryption' -import { pOnce } from '../../utils' +import ServerPersistentStore from './ServerPersistentStore' -interface Storage { +export interface PersistentStore { get(key: K): Promise set(key: K, value: V): Promise has(key: K): Promise @@ -16,95 +10,6 @@ interface Storage { size(): Promise } -class ServerStorage implements Storage { - readonly clientId: string - readonly streamId: string - readonly dbFilePath: string - private store?: Database - private error?: Error - - constructor({ clientId, streamId }: { clientId: string, streamId: string }) { - this.streamId = encodeURIComponent(streamId) - this.clientId = encodeURIComponent(clientId) - const paths = envPaths('streamr-client') - const dbFilePath = join(paths.data, clientId, 'GroupKeys.db') - this.dbFilePath = dbFilePath - - this.init = pOnce(this.init.bind(this)) - } - - async init() { - try { - await fs.mkdir(dirname(this.dbFilePath), { recursive: true }) - // open the database - const store = await open({ - filename: this.dbFilePath, - driver: sqlite3.Database - }) - await store.exec(`CREATE TABLE IF NOT EXISTS GroupKeys ( - id TEXT, - groupKey TEXT, - streamId TEXT - )`) - await store.exec('CREATE UNIQUE INDEX IF NOT EXISTS name ON GroupKeys (id)') - this.store = store - } catch (err) { - if (!this.error) { - this.error = err - } - } - - if (this.error) { - throw this.error - } - } - - async get(key: string) { - await this.init() - const value = await this.store!.get('SELECT groupKey FROM GroupKeys WHERE id = ? AND streamId = ?', key, this.streamId) - return value?.groupKey - } - - async has(key: string) { - await this.init() - const value = await this.store!.get('SELECT COUNT(*) FROM GroupKeys WHERE id = ? AND streamId = ?', key, this.streamId) - return value && value['COUNT(*)'] != null && value['COUNT(*)'] !== 0 - } - - async set(key: string, value: string) { - await this.init() - const result = await this.store!.run('INSERT INTO GroupKeys VALUES ($id, $groupKey, $streamId) ON CONFLICT DO NOTHING', { - $id: key, - $groupKey: value, - $streamId: this.streamId, - }) - - return !!result?.changes - } - - async delete(key: string) { - await this.init() - const result = await this.store!.run('DELETE FROM GroupKeys WHERE id = ? AND streamId = ?', key, this.streamId) - return !!result?.changes - } - - async clear() { - await this.init() - const result = await this.store!.run('DELETE FROM GroupKeys WHERE streamId = ?', this.streamId) - return !!result?.changes - } - - async size() { - await this.init() - const size = await this.store!.get('SELECT COUNT(*) FROM GroupKeys WHERE streamId = ?;', this.streamId) - return size && size['COUNT(*)'] - } - - get [Symbol.toStringTag]() { - return this.constructor.name - } -} - type GroupKeyId = string type GroupKeyStoreOptions = { @@ -114,9 +19,9 @@ type GroupKeyStoreOptions = { } export class GroupKeyPersistence { - store: Storage + store: PersistentStore constructor({ clientId, streamId }: { clientId: string, streamId: string }) { - this.store = new ServerStorage({ clientId, streamId }) + this.store = new ServerPersistentStore({ clientId, streamId }) } async has(groupKeyId: string) { diff --git a/src/stream/encryption/ServerPersistentStore.ts b/src/stream/encryption/ServerPersistentStore.ts new file mode 100644 index 000000000..e348b20d1 --- /dev/null +++ b/src/stream/encryption/ServerPersistentStore.ts @@ -0,0 +1,97 @@ +import envPaths from 'env-paths' +import { dirname, join } from 'path' +import { promises as fs } from 'fs' +import { open, Database } from 'sqlite' +import sqlite3 from 'sqlite3' + +import { PersistentStore } from './GroupKeyStore' +import { pOnce } from '../../utils' + +export default class ServerPersistentStore implements PersistentStore { + readonly clientId: string + readonly streamId: string + readonly dbFilePath: string + private store?: Database + private error?: Error + + constructor({ clientId, streamId }: { clientId: string, streamId: string }) { + this.streamId = encodeURIComponent(streamId) + this.clientId = encodeURIComponent(clientId) + const paths = envPaths('streamr-client') + const dbFilePath = join(paths.data, clientId, 'GroupKeys.db') + this.dbFilePath = dbFilePath + + this.init = pOnce(this.init.bind(this)) + } + + async init() { + try { + await fs.mkdir(dirname(this.dbFilePath), { recursive: true }) + // open the database + const store = await open({ + filename: this.dbFilePath, + driver: sqlite3.Database + }) + await store.exec(`CREATE TABLE IF NOT EXISTS GroupKeys ( + id TEXT, + groupKey TEXT, + streamId TEXT + )`) + await store.exec('CREATE UNIQUE INDEX IF NOT EXISTS name ON GroupKeys (id)') + this.store = store + } catch (err) { + if (!this.error) { + this.error = err + } + } + + if (this.error) { + throw this.error + } + } + + async get(key: string) { + await this.init() + const value = await this.store!.get('SELECT groupKey FROM GroupKeys WHERE id = ? AND streamId = ?', key, this.streamId) + return value?.groupKey + } + + async has(key: string) { + await this.init() + const value = await this.store!.get('SELECT COUNT(*) FROM GroupKeys WHERE id = ? AND streamId = ?', key, this.streamId) + return value && value['COUNT(*)'] != null && value['COUNT(*)'] !== 0 + } + + async set(key: string, value: string) { + await this.init() + const result = await this.store!.run('INSERT INTO GroupKeys VALUES ($id, $groupKey, $streamId) ON CONFLICT DO NOTHING', { + $id: key, + $groupKey: value, + $streamId: this.streamId, + }) + + return !!result?.changes + } + + async delete(key: string) { + await this.init() + const result = await this.store!.run('DELETE FROM GroupKeys WHERE id = ? AND streamId = ?', key, this.streamId) + return !!result?.changes + } + + async clear() { + await this.init() + const result = await this.store!.run('DELETE FROM GroupKeys WHERE streamId = ?', this.streamId) + return !!result?.changes + } + + async size() { + await this.init() + const size = await this.store!.get('SELECT COUNT(*) FROM GroupKeys WHERE streamId = ?;', this.streamId) + return size && size['COUNT(*)'] + } + + get [Symbol.toStringTag]() { + return this.constructor.name + } +} diff --git a/src/stream/encryption/ServerStore.ts b/src/stream/encryption/ServerStore.ts deleted file mode 100644 index e69de29bb..000000000 diff --git a/webpack.config.js b/webpack.config.js index 500fb67c7..2ca3caa7e 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -103,8 +103,10 @@ module.exports = (env, argv) => { 'node-fetch': path.resolve(__dirname, './src/shim/node-fetch.js'), 'node-webcrypto-ossl': path.resolve(__dirname, 'src/shim/crypto.js'), 'streamr-client-protocol': path.resolve(__dirname, 'node_modules/streamr-client-protocol/dist/src'), - // swap out PersistentStore for browser - [path.resolve(__dirname, 'src/stream/PersistentStore')]: path.resolve(__dirname, 'src/stream/BrowserStore'), + // swap out ServerPersistentStore for BrowserPersistentStore + [path.resolve(__dirname, 'src/stream/encryption/ServerPersistentStore')]: ( + path.resolve(__dirname, 'src/stream/encryption/BrowserPersistentStore') + ), } }, plugins: [ From aeb68efacf7b1f4857c7feced29c43b65cd77195 Mon Sep 17 00:00:00 2001 From: Tim Oxley Date: Thu, 6 May 2021 16:25:05 -0400 Subject: [PATCH 21/28] refactor: Split key exchange publisher/subscriber into separate files. --- src/publish/Encrypt.ts | 2 +- src/rest/StreamEndpoints.ts | 2 +- src/stream/encryption/KeyExchangePublisher.ts | 196 ++++++++++++++ ...eyExchange.ts => KeyExchangeSubscriber.ts} | 256 +----------------- src/stream/encryption/KeyExchangeUtils.ts | 74 +++++ src/subscribe/Decrypt.js | 2 +- 6 files changed, 284 insertions(+), 248 deletions(-) create mode 100644 src/stream/encryption/KeyExchangePublisher.ts rename src/stream/encryption/{KeyExchange.ts => KeyExchangeSubscriber.ts} (54%) create mode 100644 src/stream/encryption/KeyExchangeUtils.ts diff --git a/src/publish/Encrypt.ts b/src/publish/Encrypt.ts index a191a2231..617730e16 100644 --- a/src/publish/Encrypt.ts +++ b/src/publish/Encrypt.ts @@ -3,7 +3,7 @@ import { MessageLayer } from 'streamr-client-protocol' import EncryptionUtil from '../stream/encryption/Encryption' import { Stream } from '../stream' import { StreamrClient } from '../StreamrClient' -import { PublisherKeyExhange } from '../stream/encryption/KeyExchange' +import { PublisherKeyExhange } from '../stream/encryption/KeyExchangePublisher' const { StreamMessage } = MessageLayer diff --git a/src/rest/StreamEndpoints.ts b/src/rest/StreamEndpoints.ts index 51e0f9b9c..972afb905 100644 --- a/src/rest/StreamEndpoints.ts +++ b/src/rest/StreamEndpoints.ts @@ -8,7 +8,7 @@ import { getEndpointUrl } from '../utils' import { validateOptions } from '../stream/utils' import { Stream, StreamOperation, StreamProperties } from '../stream' import { StreamPart } from '../stream/StreamPart' -import { isKeyExchangeStream } from '../stream/encryption/KeyExchange' +import { isKeyExchangeStream } from '../stream/encryption/KeyExchangeUtils' import authFetch, { ErrorCode, NotFoundError } from './authFetch' import { EthereumAddress } from '../types' diff --git a/src/stream/encryption/KeyExchangePublisher.ts b/src/stream/encryption/KeyExchangePublisher.ts new file mode 100644 index 000000000..c3e0c4279 --- /dev/null +++ b/src/stream/encryption/KeyExchangePublisher.ts @@ -0,0 +1,196 @@ +import { + StreamMessage, GroupKeyRequest, GroupKeyResponse, EncryptedGroupKey, GroupKeyErrorResponse, Errors +} from 'streamr-client-protocol' +import pMemoize from 'p-memoize' + +import Scaffold from '../../utils/Scaffold' + +import { validateOptions } from '../utils' +import EncryptionUtil, { GroupKey, StreamMessageProcessingError } from './Encryption' +import type { Subscription } from '../../subscribe' +import { StreamrClient } from '../../StreamrClient' +import GroupKeyStore from './GroupKeyStore' +import { + subscribeToKeyExchangeStream, + parseGroupKeys, + getKeyExchangeStreamId, + GroupKeysSerialized +} from './KeyExchangeUtils' + +const { ValidationError } = Errors + +class InvalidGroupKeyRequestError extends ValidationError { + code: string + constructor(...args: ConstructorParameters) { + super(...args) + this.code = 'INVALID_GROUP_KEY_REQUEST' + if (Error.captureStackTrace) { + Error.captureStackTrace(this, this.constructor) + } + } +} + +async function catchKeyExchangeError(client: StreamrClient, streamMessage: StreamMessage, fn: (...args: any[]) => Promise) { + try { + return await fn() + } catch (error) { + const subscriberId = streamMessage.getPublisherId() + const msg = streamMessage.getParsedContent() + const { streamId, requestId, groupKeyIds } = GroupKeyRequest.fromArray(msg) + return client.publish(getKeyExchangeStreamId(subscriberId), new GroupKeyErrorResponse({ + requestId, + streamId, + errorCode: error.code || 'UNEXPECTED_ERROR', + errorMessage: error.message, + groupKeyIds + })) + } +} + +async function PublisherKeyExhangeSubscription(client: StreamrClient, getGroupKeyStore: (streamId: string) => Promise) { + async function onKeyExchangeMessage(_parsedContent: any, streamMessage: StreamMessage) { + return catchKeyExchangeError(client, streamMessage, async () => { + if (streamMessage.messageType !== StreamMessage.MESSAGE_TYPES.GROUP_KEY_REQUEST) { + return Promise.resolve() + } + + // No need to check if parsedContent contains the necessary fields because it was already checked during deserialization + const { requestId, streamId, rsaPublicKey, groupKeyIds } = GroupKeyRequest.fromArray(streamMessage.getParsedContent()) + + const subscriberId = streamMessage.getPublisherId() + + const groupKeyStore = await getGroupKeyStore(streamId) + const isSubscriber = await client.isStreamSubscriber(streamId, subscriberId) + const encryptedGroupKeys = (!isSubscriber ? [] : await Promise.all(groupKeyIds.map(async (id) => { + const groupKey = await groupKeyStore.get(id) + if (!groupKey) { + return null // will be filtered out + } + const key = EncryptionUtil.encryptWithPublicKey(groupKey.data, rsaPublicKey, true) + return new EncryptedGroupKey(id, key) + }))).filter(Boolean) as EncryptedGroupKey[] + + client.debug('Publisher: Subscriber requested groupKeys: %d. Got: %d. %o', groupKeyIds.length, encryptedGroupKeys.length, { + subscriberId, + groupKeyIds, + responseKeys: encryptedGroupKeys.map(({ groupKeyId }) => groupKeyId), + }) + + const response = new GroupKeyResponse({ + streamId, + requestId, + encryptedGroupKeys, + }) + + // hack overriding toStreamMessage method to set correct encryption type + const toStreamMessage = response.toStreamMessage.bind(response) + response.toStreamMessage = (...args) => { + const msg = toStreamMessage(...args) + msg.encryptionType = StreamMessage.ENCRYPTION_TYPES.RSA + return msg + } + + return client.publish(getKeyExchangeStreamId(subscriberId), response) + }) + } + + const sub = await subscribeToKeyExchangeStream(client, onKeyExchangeMessage) + sub.on('error', (err: Error | StreamMessageProcessingError) => { + if (!('streamMessage' in err)) { + return // do nothing + } + + // wrap error and translate into ErrorResponse. + catchKeyExchangeError(client, err.streamMessage, () => { // eslint-disable-line promise/no-promise-in-callback + // rethrow so catchKeyExchangeError handles it + throw new InvalidGroupKeyRequestError(err.message) + }).catch((unexpectedError) => { + sub.emit('error', unexpectedError) + }) + }) + + return sub +} + +type KeyExhangeOptions = { + groupKeys?: Record +} + +export function PublisherKeyExhange(client: StreamrClient, { groupKeys = {} }: KeyExhangeOptions = {}) { + let enabled = true + const getGroupKeyStore = pMemoize(async (streamId) => { + const clientId = await client.getAddress() + return new GroupKeyStore({ + clientId, + streamId, + groupKeys: [...parseGroupKeys(groupKeys[streamId]).entries()] + }) + }, { + cacheKey([maybeStreamId]) { + const { streamId } = validateOptions(maybeStreamId) + return streamId + } + }) + + let sub: Subscription | undefined + const next = Scaffold([ + async () => { + sub = await PublisherKeyExhangeSubscription(client, getGroupKeyStore) + return async () => { + if (!sub) { return } + const cancelTask = sub.cancel() + sub = undefined + await cancelTask + } + } + ], async () => enabled) + + async function rotateGroupKey(streamId: string) { + if (!enabled) { return } + const groupKeyStore = await getGroupKeyStore(streamId) + await groupKeyStore.rotateGroupKey() + } + + async function setNextGroupKey(streamId: string, groupKey: GroupKey) { + if (!enabled) { return } + const groupKeyStore = await getGroupKeyStore(streamId) + + await groupKeyStore.setNextGroupKey(groupKey) + } + + async function useGroupKey(streamId: string) { + await next() + if (!enabled) { return [] } + const groupKeyStore = await getGroupKeyStore(streamId) + return groupKeyStore.useGroupKey() + } + + async function hasAnyGroupKey(streamId: string) { + const groupKeyStore = await getGroupKeyStore(streamId) + return !groupKeyStore.isEmpty() + } + + async function rekey(streamId: string) { + if (!enabled) { return } + const groupKeyStore = await getGroupKeyStore(streamId) + await groupKeyStore.rekey() + await next() + } + + return { + setNextGroupKey, + useGroupKey, + rekey, + rotateGroupKey, + hasAnyGroupKey, + async start() { + enabled = true + return next() + }, + async stop() { + pMemoize.clear(getGroupKeyStore) + enabled = false + return next() + } + } +} diff --git a/src/stream/encryption/KeyExchange.ts b/src/stream/encryption/KeyExchangeSubscriber.ts similarity index 54% rename from src/stream/encryption/KeyExchange.ts rename to src/stream/encryption/KeyExchangeSubscriber.ts index 844034b73..7fc0432a3 100644 --- a/src/stream/encryption/KeyExchange.ts +++ b/src/stream/encryption/KeyExchangeSubscriber.ts @@ -1,77 +1,23 @@ import { - StreamMessage, GroupKeyRequest, GroupKeyResponse, GroupKeyErrorResponse, EncryptedGroupKey, Errors + StreamMessage, GroupKeyRequest, GroupKeyResponse } from 'streamr-client-protocol' -import mem from 'mem' import pMemoize from 'p-memoize' - import { uuid, Defer } from '../../utils' import Scaffold from '../../utils/Scaffold' +import mem from 'mem' import { validateOptions } from '../utils' -import EncryptionUtil, { GroupKey, GroupKeyish, StreamMessageProcessingError } from './Encryption' +import EncryptionUtil, { GroupKey } from './Encryption' import type { Subscription } from '../../subscribe' import { StreamrClient } from '../../StreamrClient' import GroupKeyStore from './GroupKeyStore' - -const KEY_EXCHANGE_STREAM_PREFIX = 'SYSTEM/keyexchange' - -const { ValidationError } = Errors - -export function isKeyExchangeStream(id = '') { - return id.startsWith(KEY_EXCHANGE_STREAM_PREFIX) -} - -class InvalidGroupKeyRequestError extends ValidationError { - code: string - constructor(...args: ConstructorParameters) { - super(...args) - this.code = 'INVALID_GROUP_KEY_REQUEST' - if (Error.captureStackTrace) { - Error.captureStackTrace(this, this.constructor) - } - } -} - -/* -class InvalidGroupKeyResponseError extends Error { - constructor(...args) { - super(...args) - this.code = 'INVALID_GROUP_KEY_RESPONSE' - if (Error.captureStackTrace) { - Error.captureStackTrace(this, this.constructor) - } - } -} - -class InvalidContentTypeError extends Error { - constructor(...args) { - super(...args) - this.code = 'INVALID_MESSAGE_TYPE' - if (Error.captureStackTrace) { - Error.captureStackTrace(this, this.constructor) - } - } -} -*/ - -type Address = string -type GroupKeyId = string - -function getKeyExchangeStreamId(address: Address) { - if (isKeyExchangeStream(address)) { - return address // prevent ever double-handling - } - return `${KEY_EXCHANGE_STREAM_PREFIX}/${address.toLowerCase()}` -} - -type GroupKeysSerialized = Record - -function parseGroupKeys(groupKeys: GroupKeysSerialized = {}): Map { - return new Map(Object.entries(groupKeys || {}).map(([key, value]) => { - if (!value || !key) { return null } - return [key, GroupKey.from(value)] - }).filter(Boolean) as []) -} +import { + GroupKeyId, + subscribeToKeyExchangeStream, + parseGroupKeys, + getKeyExchangeStreamId, + KeyExhangeOptions, +} from './KeyExchangeUtils' type MessageMatch = (content: any, streamMessage: StreamMessage) => boolean @@ -95,186 +41,6 @@ function waitForSubMessage(sub: Subscription, matchFn: MessageMatch) { return task } -async function subscribeToKeyExchangeStream(client: StreamrClient, onKeyExchangeMessage: (msg: any, streamMessage: StreamMessage) => void) { - const { options } = client - if ((!options.auth!.privateKey && !options.auth!.ethereum) || !options.keyExchange) { - return Promise.resolve() - } - - await client.session.getSessionToken() // trigger auth errors if any - // subscribing to own keyexchange stream - const publisherId = await client.getUserId() - const streamId = getKeyExchangeStreamId(publisherId) - const sub = await client.subscribe(streamId, onKeyExchangeMessage) - sub.on('error', () => {}) // errors should not shut down subscription - return sub -} - -async function catchKeyExchangeError(client: StreamrClient, streamMessage: StreamMessage, fn: (...args: any[]) => Promise) { - try { - return await fn() - } catch (error) { - const subscriberId = streamMessage.getPublisherId() - const msg = streamMessage.getParsedContent() - const { streamId, requestId, groupKeyIds } = GroupKeyRequest.fromArray(msg) - return client.publish(getKeyExchangeStreamId(subscriberId), new GroupKeyErrorResponse({ - requestId, - streamId, - errorCode: error.code || 'UNEXPECTED_ERROR', - errorMessage: error.message, - groupKeyIds - })) - } -} - -async function PublisherKeyExhangeSubscription(client: StreamrClient, getGroupKeyStore: (streamId: string) => Promise) { - async function onKeyExchangeMessage(_parsedContent: any, streamMessage: StreamMessage) { - return catchKeyExchangeError(client, streamMessage, async () => { - if (streamMessage.messageType !== StreamMessage.MESSAGE_TYPES.GROUP_KEY_REQUEST) { - return Promise.resolve() - } - - // No need to check if parsedContent contains the necessary fields because it was already checked during deserialization - const { requestId, streamId, rsaPublicKey, groupKeyIds } = GroupKeyRequest.fromArray(streamMessage.getParsedContent()) - - const subscriberId = streamMessage.getPublisherId() - - const groupKeyStore = await getGroupKeyStore(streamId) - const isSubscriber = await client.isStreamSubscriber(streamId, subscriberId) - const encryptedGroupKeys = (!isSubscriber ? [] : await Promise.all(groupKeyIds.map(async (id) => { - const groupKey = await groupKeyStore.get(id) - if (!groupKey) { - return null // will be filtered out - } - const key = EncryptionUtil.encryptWithPublicKey(groupKey.data, rsaPublicKey, true) - return new EncryptedGroupKey(id, key) - }))).filter(Boolean) as EncryptedGroupKey[] - - client.debug('Publisher: Subscriber requested groupKeys: %d. Got: %d. %o', groupKeyIds.length, encryptedGroupKeys.length, { - subscriberId, - groupKeyIds, - responseKeys: encryptedGroupKeys.map(({ groupKeyId }) => groupKeyId), - }) - - const response = new GroupKeyResponse({ - streamId, - requestId, - encryptedGroupKeys, - }) - - // hack overriding toStreamMessage method to set correct encryption type - const toStreamMessage = response.toStreamMessage.bind(response) - response.toStreamMessage = (...args) => { - const msg = toStreamMessage(...args) - msg.encryptionType = StreamMessage.ENCRYPTION_TYPES.RSA - return msg - } - - return client.publish(getKeyExchangeStreamId(subscriberId), response) - }) - } - - const sub = await subscribeToKeyExchangeStream(client, onKeyExchangeMessage) - sub.on('error', (err: Error | StreamMessageProcessingError) => { - if (!('streamMessage' in err)) { - return // do nothing - } - - // wrap error and translate into ErrorResponse. - catchKeyExchangeError(client, err.streamMessage, () => { // eslint-disable-line promise/no-promise-in-callback - // rethrow so catchKeyExchangeError handles it - throw new InvalidGroupKeyRequestError(err.message) - }).catch((unexpectedError) => { - sub.emit('error', unexpectedError) - }) - }) - - return sub -} - -type KeyExhangeOptions = { - groupKeys?: Record -} - -export function PublisherKeyExhange(client: StreamrClient, { groupKeys = {} }: KeyExhangeOptions = {}) { - let enabled = true - const getGroupKeyStore = pMemoize(async (streamId) => { - const clientId = await client.getAddress() - return new GroupKeyStore({ - clientId, - streamId, - groupKeys: [...parseGroupKeys(groupKeys[streamId]).entries()] - }) - }, { - cacheKey([maybeStreamId]) { - const { streamId } = validateOptions(maybeStreamId) - return streamId - } - }) - - let sub: Subscription | undefined - const next = Scaffold([ - async () => { - sub = await PublisherKeyExhangeSubscription(client, getGroupKeyStore) - return async () => { - if (!sub) { return } - const cancelTask = sub.cancel() - sub = undefined - await cancelTask - } - } - ], async () => enabled) - - async function rotateGroupKey(streamId: string) { - if (!enabled) { return } - const groupKeyStore = await getGroupKeyStore(streamId) - await groupKeyStore.rotateGroupKey() - } - - async function setNextGroupKey(streamId: string, groupKey: GroupKey) { - if (!enabled) { return } - const groupKeyStore = await getGroupKeyStore(streamId) - - await groupKeyStore.setNextGroupKey(groupKey) - } - - async function useGroupKey(streamId: string) { - await next() - if (!enabled) { return [] } - const groupKeyStore = await getGroupKeyStore(streamId) - return groupKeyStore.useGroupKey() - } - - async function hasAnyGroupKey(streamId: string) { - const groupKeyStore = await getGroupKeyStore(streamId) - return !groupKeyStore.isEmpty() - } - - async function rekey(streamId: string) { - if (!enabled) { return } - const groupKeyStore = await getGroupKeyStore(streamId) - await groupKeyStore.rekey() - await next() - } - - return { - setNextGroupKey, - useGroupKey, - rekey, - rotateGroupKey, - hasAnyGroupKey, - async start() { - enabled = true - return next() - }, - async stop() { - pMemoize.clear(getGroupKeyStore) - enabled = false - return next() - } - } -} - async function getGroupKeysFromStreamMessage(streamMessage: StreamMessage, encryptionUtil: EncryptionUtil) { const { encryptedGroupKeys } = GroupKeyResponse.fromArray(streamMessage.getParsedContent()) return Promise.all(encryptedGroupKeys.map(async (encryptedGroupKey) => ( @@ -540,7 +306,7 @@ export function SubscriberKeyExchange(client: StreamrClient, { groupKeys = {} }: const streamId = streamMessage.getStreamId() const groupKeyStore = await getGroupKeyStore(streamId) // newGroupKey has been converted into GroupKey - const newGroupKey: unknown = streamMessage.newGroupKey + const { newGroupKey } = streamMessage await groupKeyStore.add(newGroupKey as GroupKey) }, async stop() { diff --git a/src/stream/encryption/KeyExchangeUtils.ts b/src/stream/encryption/KeyExchangeUtils.ts new file mode 100644 index 000000000..98d392071 --- /dev/null +++ b/src/stream/encryption/KeyExchangeUtils.ts @@ -0,0 +1,74 @@ +import { + StreamMessage, Errors +} from 'streamr-client-protocol' + +import { GroupKey, GroupKeyish } from './Encryption' +import { StreamrClient } from '../../StreamrClient' + +const KEY_EXCHANGE_STREAM_PREFIX = 'SYSTEM/keyexchange' + +export const { ValidationError } = Errors + +export function isKeyExchangeStream(id = '') { + return id.startsWith(KEY_EXCHANGE_STREAM_PREFIX) +} + +/* +class InvalidGroupKeyResponseError extends Error { + constructor(...args) { + super(...args) + this.code = 'INVALID_GROUP_KEY_RESPONSE' + if (Error.captureStackTrace) { + Error.captureStackTrace(this, this.constructor) + } + } +} + +class InvalidContentTypeError extends Error { + constructor(...args) { + super(...args) + this.code = 'INVALID_MESSAGE_TYPE' + if (Error.captureStackTrace) { + Error.captureStackTrace(this, this.constructor) + } + } +} +*/ + +type Address = string +export type GroupKeyId = string + +export function getKeyExchangeStreamId(address: Address) { + if (isKeyExchangeStream(address)) { + return address // prevent ever double-handling + } + return `${KEY_EXCHANGE_STREAM_PREFIX}/${address.toLowerCase()}` +} + +export type GroupKeysSerialized = Record + +export function parseGroupKeys(groupKeys: GroupKeysSerialized = {}): Map { + return new Map(Object.entries(groupKeys || {}).map(([key, value]) => { + if (!value || !key) { return null } + return [key, GroupKey.from(value)] + }).filter(Boolean) as []) +} + +export async function subscribeToKeyExchangeStream(client: StreamrClient, onKeyExchangeMessage: (msg: any, streamMessage: StreamMessage) => void) { + const { options } = client + if ((!options.auth!.privateKey && !options.auth!.ethereum) || !options.keyExchange) { + return Promise.resolve() + } + + await client.session.getSessionToken() // trigger auth errors if any + // subscribing to own keyexchange stream + const publisherId = await client.getUserId() + const streamId = getKeyExchangeStreamId(publisherId) + const sub = await client.subscribe(streamId, onKeyExchangeMessage) + sub.on('error', () => {}) // errors should not shut down subscription + return sub +} + +export type KeyExhangeOptions = { + groupKeys?: Record +} diff --git a/src/subscribe/Decrypt.js b/src/subscribe/Decrypt.js index 2f4d3d296..c51a6d66c 100644 --- a/src/subscribe/Decrypt.js +++ b/src/subscribe/Decrypt.js @@ -1,7 +1,7 @@ import { MessageLayer } from 'streamr-client-protocol' import EncryptionUtil, { UnableToDecryptError } from '../stream/encryption/Encryption' -import { SubscriberKeyExchange } from '../stream/encryption/KeyExchange' +import { SubscriberKeyExchange } from '../stream/encryption/KeyExchangeSubscriber' const { StreamMessage } = MessageLayer From 3b809ef229769bcace4400517e73bf95d72916d7 Mon Sep 17 00:00:00 2001 From: Tim Oxley Date: Thu, 6 May 2021 17:10:49 -0400 Subject: [PATCH 22/28] refactor: Convert key exchange publisher/subscriber into classes. --- src/publish/Encrypt.ts | 12 +- src/stream/encryption/KeyExchangePublisher.ts | 112 +++---- .../encryption/KeyExchangeSubscriber.ts | 281 ++++++++++-------- src/subscribe/Decrypt.js | 8 +- 4 files changed, 216 insertions(+), 197 deletions(-) diff --git a/src/publish/Encrypt.ts b/src/publish/Encrypt.ts index 617730e16..5b8492aaf 100644 --- a/src/publish/Encrypt.ts +++ b/src/publish/Encrypt.ts @@ -7,14 +7,12 @@ import { PublisherKeyExhange } from '../stream/encryption/KeyExchangePublisher' const { StreamMessage } = MessageLayer -type PublisherKeyExhangeAPI = ReturnType - export default function Encrypt(client: StreamrClient) { - let publisherKeyExchange: ReturnType + let publisherKeyExchange: PublisherKeyExhange function getPublisherKeyExchange() { if (!publisherKeyExchange) { - publisherKeyExchange = PublisherKeyExhange(client, { + publisherKeyExchange = new PublisherKeyExhange(client, { groupKeys: { ...client.options.groupKeys, } @@ -59,13 +57,13 @@ export default function Encrypt(client: StreamrClient) { } return Object.assign(encrypt, { - setNextGroupKey(...args: Parameters) { + setNextGroupKey(...args: Parameters) { return getPublisherKeyExchange().setNextGroupKey(...args) }, - rotateGroupKey(...args: Parameters) { + rotateGroupKey(...args: Parameters) { return getPublisherKeyExchange().rotateGroupKey(...args) }, - rekey(...args: Parameters) { + rekey(...args: Parameters) { return getPublisherKeyExchange().rekey(...args) }, start() { diff --git a/src/stream/encryption/KeyExchangePublisher.ts b/src/stream/encryption/KeyExchangePublisher.ts index c3e0c4279..c452ce684 100644 --- a/src/stream/encryption/KeyExchangePublisher.ts +++ b/src/stream/encryption/KeyExchangePublisher.ts @@ -116,81 +116,83 @@ type KeyExhangeOptions = { groupKeys?: Record } -export function PublisherKeyExhange(client: StreamrClient, { groupKeys = {} }: KeyExhangeOptions = {}) { - let enabled = true - const getGroupKeyStore = pMemoize(async (streamId) => { - const clientId = await client.getAddress() +export class PublisherKeyExhange { + enabled = true + next + client + initialGroupKeys + constructor(client: StreamrClient, { groupKeys = {} }: KeyExhangeOptions = {}) { + this.client = client + this.initialGroupKeys = groupKeys + this.getGroupKeyStore = pMemoize(this.getGroupKeyStore.bind(this), { + cacheKey([maybeStreamId]) { + const { streamId } = validateOptions(maybeStreamId) + return streamId + } + }) + + let sub: Subscription | undefined + this.next = Scaffold([ + async () => { + sub = await PublisherKeyExhangeSubscription(client, this.getGroupKeyStore) + return async () => { + if (!sub) { return } + const cancelTask = sub.cancel() + sub = undefined + await cancelTask + } + } + ], async () => this.enabled) + } + + async getGroupKeyStore(streamId: string) { + const clientId = await this.client.getAddress() return new GroupKeyStore({ clientId, streamId, - groupKeys: [...parseGroupKeys(groupKeys[streamId]).entries()] + groupKeys: [...parseGroupKeys(this.initialGroupKeys[streamId]).entries()] }) - }, { - cacheKey([maybeStreamId]) { - const { streamId } = validateOptions(maybeStreamId) - return streamId - } - }) - - let sub: Subscription | undefined - const next = Scaffold([ - async () => { - sub = await PublisherKeyExhangeSubscription(client, getGroupKeyStore) - return async () => { - if (!sub) { return } - const cancelTask = sub.cancel() - sub = undefined - await cancelTask - } - } - ], async () => enabled) + } - async function rotateGroupKey(streamId: string) { - if (!enabled) { return } - const groupKeyStore = await getGroupKeyStore(streamId) + async rotateGroupKey(streamId: string) { + if (!this.enabled) { return } + const groupKeyStore = await this.getGroupKeyStore(streamId) await groupKeyStore.rotateGroupKey() } - async function setNextGroupKey(streamId: string, groupKey: GroupKey) { - if (!enabled) { return } - const groupKeyStore = await getGroupKeyStore(streamId) + async setNextGroupKey(streamId: string, groupKey: GroupKey) { + if (!this.enabled) { return } + const groupKeyStore = await this.getGroupKeyStore(streamId) await groupKeyStore.setNextGroupKey(groupKey) } - async function useGroupKey(streamId: string) { - await next() - if (!enabled) { return [] } - const groupKeyStore = await getGroupKeyStore(streamId) + async useGroupKey(streamId: string) { + await this.next() + if (!this.enabled) { return [] } + const groupKeyStore = await this.getGroupKeyStore(streamId) return groupKeyStore.useGroupKey() } - async function hasAnyGroupKey(streamId: string) { - const groupKeyStore = await getGroupKeyStore(streamId) + async hasAnyGroupKey(streamId: string) { + const groupKeyStore = await this.getGroupKeyStore(streamId) return !groupKeyStore.isEmpty() } - async function rekey(streamId: string) { - if (!enabled) { return } - const groupKeyStore = await getGroupKeyStore(streamId) + async rekey(streamId: string) { + if (!this.enabled) { return } + const groupKeyStore = await this.getGroupKeyStore(streamId) await groupKeyStore.rekey() - await next() + await this.next() + } + async start() { + this.enabled = true + return this.next() } - return { - setNextGroupKey, - useGroupKey, - rekey, - rotateGroupKey, - hasAnyGroupKey, - async start() { - enabled = true - return next() - }, - async stop() { - pMemoize.clear(getGroupKeyStore) - enabled = false - return next() - } + async stop() { + pMemoize.clear(this.getGroupKeyStore) + this.enabled = false + return this.next() } } diff --git a/src/stream/encryption/KeyExchangeSubscriber.ts b/src/stream/encryption/KeyExchangeSubscriber.ts index 7fc0432a3..176001208 100644 --- a/src/stream/encryption/KeyExchangeSubscriber.ts +++ b/src/stream/encryption/KeyExchangeSubscriber.ts @@ -80,44 +80,50 @@ async function SubscriberKeyExhangeSubscription( return sub } -export function SubscriberKeyExchange(client: StreamrClient, { groupKeys = {} }: KeyExhangeOptions = {}) { - let enabled = true - const encryptionUtil = new EncryptionUtil(client.options.keyExchange) +export class SubscriberKeyExchange { + requestKeysStep?: () => Promise + client + initialGroupKeys + encryptionUtil + pending = new Map>() + getBuffer = mem<(groupKeyId: GroupKeyId) => GroupKeyId[], [string]>(() => []) + timeouts: Record> = Object.create(null) + next + enabled = true + sub?: Subscription - const getGroupKeyStore = pMemoize(async (streamId) => { - const clientId = await client.getAddress() - return new GroupKeyStore({ - clientId, - streamId, - groupKeys: [...parseGroupKeys(groupKeys[streamId]).entries()] + constructor(client: StreamrClient, { groupKeys = {} }: KeyExhangeOptions = {}) { + this.client = client + this.initialGroupKeys = groupKeys + this.getGroupKeyStore = pMemoize(this.getGroupKeyStore.bind(this), { + cacheKey([maybeStreamId]) { + const { streamId } = validateOptions(maybeStreamId) + return streamId + } }) - }, { - cacheKey([maybeStreamId]) { - const { streamId } = validateOptions(maybeStreamId) - return streamId - } - }) + this.encryptionUtil = new EncryptionUtil(client.options.keyExchange) + this.next = this.initNext() + } - let sub: Subscription | undefined - let requestKeysStep: () => Promise - async function requestKeys({ streamId, publisherId, groupKeyIds }: { + async requestKeys({ streamId, publisherId, groupKeyIds }: { streamId: string, publisherId: string, groupKeyIds: GroupKeyId[] }) { let done = false const requestId = uuid('GroupKeyRequest') - const rsaPublicKey = encryptionUtil.getPublicKey() + const rsaPublicKey = this.encryptionUtil.getPublicKey() const keyExchangeStreamId = getKeyExchangeStreamId(publisherId) let responseTask: ReturnType let cancelTask: ReturnType let receivedGroupKeys: GroupKey[] = [] let response: any - const step = Scaffold([ + + this.requestKeysStep = Scaffold([ async () => { - if (!sub) { throw new Error('no subscription') } + if (!this.sub) { throw new Error('no subscription') } cancelTask = Defer() - responseTask = waitForSubMessage(sub, (content, streamMessage) => { + responseTask = waitForSubMessage(this.sub, (content, streamMessage) => { const { messageType } = streamMessage const matchesMessageType = ( messageType === StreamMessage.MESSAGE_TYPES.GROUP_KEY_RESPONSE @@ -143,20 +149,20 @@ export function SubscriberKeyExchange(client: StreamrClient, { groupKeys = {} }: rsaPublicKey, groupKeyIds, }) - await client.publish(keyExchangeStreamId, msg) + await this.client.publish(keyExchangeStreamId, msg) }, async () => { response = await responseTask return () => { response = undefined } }, async () => { - receivedGroupKeys = response ? await getGroupKeysFromStreamMessage(response, encryptionUtil) : [] + receivedGroupKeys = response ? await getGroupKeysFromStreamMessage(response, this.encryptionUtil) : [] return () => { receivedGroupKeys = [] } }, - ], async () => enabled && !done, { + ], async () => this.enabled && !done, { id: `requestKeys.${requestId}`, onChange(isGoingUp) { if (!isGoingUp && cancelTask) { @@ -165,153 +171,166 @@ export function SubscriberKeyExchange(client: StreamrClient, { groupKeys = {} }: } }) - requestKeysStep = step - await step() + await this.requestKeysStep() const keys = receivedGroupKeys.slice() done = true - await step() + await this.requestKeysStep() return keys } - const pending = new Map() - const getBuffer = mem<(groupKeyId: GroupKeyId) => GroupKeyId[], [string]>(() => []) - const timeouts: Record> = Object.create(null) + async getGroupKeyStore(streamId: string) { + const clientId = await this.client.getAddress() + return new GroupKeyStore({ + clientId, + streamId, + groupKeys: [...parseGroupKeys(this.initialGroupKeys[streamId]).entries()] + }) + } + + private async processBuffer({ streamId, publisherId }: { streamId: string, publisherId: string }) { + if (!this.enabled) { return } + const key = `${streamId}.${publisherId}` + const currentBuffer = this.getBuffer(key) + const groupKeyIds = currentBuffer.slice() + const groupKeyStore = await this.getGroupKeyStore(streamId) + currentBuffer.length = 0 + try { + const receivedGroupKeys = await this.requestKeys({ + streamId, + publisherId, + groupKeyIds, + }) + if (!this.enabled) { return } + await Promise.all(receivedGroupKeys.map(async (groupKey) => ( + groupKeyStore.add(groupKey) + ))) + if (!this.enabled) { return } + await Promise.all(groupKeyIds.map(async (id) => { + if (!this.pending.has(id)) { return } + const groupKeyTask = groupKeyStore.get(id) + const task = this.pending.get(id) + this.pending.delete(id) + const groupKey = await groupKeyTask + if (task) { + task.resolve(groupKey) + } + })) + } catch (err) { + groupKeyIds.forEach((id) => { + if (!this.pending.has(id)) { return } + const task = this.pending.get(id) + this.pending.delete(id) + if (task) { + task.reject(err) + } + }) + } + } - async function getKey(streamMessage: StreamMessage) { + async getKey(streamMessage: StreamMessage) { const streamId = streamMessage.getStreamId() const publisherId = streamMessage.getPublisherId() const { groupKeyId } = streamMessage if (!groupKeyId) { return Promise.resolve() } - const groupKeyStore = await getGroupKeyStore(streamId) - if (!enabled) { return Promise.resolve() } + const groupKeyStore = await this.getGroupKeyStore(streamId) + + if (!this.enabled) { return Promise.resolve() } const existingGroupKey = await groupKeyStore.get(groupKeyId) - if (!enabled) { return Promise.resolve() } + if (!this.enabled) { return Promise.resolve() } if (existingGroupKey) { return existingGroupKey } - if (pending.has(groupKeyId)) { - return pending.get(groupKeyId) + if (this.pending.has(groupKeyId)) { + return this.pending.get(groupKeyId) } const key = `${streamId}.${publisherId}` - const buffer = getBuffer(key) + const buffer = this.getBuffer(key) buffer.push(groupKeyId) - pending.set(groupKeyId, Defer()) - - async function processBuffer() { - if (!enabled) { return } - const currentBuffer = getBuffer(key) - const groupKeyIds = currentBuffer.slice() - currentBuffer.length = 0 - try { - const receivedGroupKeys = await requestKeys({ - streamId, - publisherId, - groupKeyIds, - }) - if (!enabled) { return } - await Promise.all(receivedGroupKeys.map(async (groupKey) => ( - groupKeyStore.add(groupKey) - ))) - if (!enabled) { return } - await Promise.all(groupKeyIds.map(async (id) => { - if (!pending.has(id)) { return } - const groupKeyTask = groupKeyStore.get(id) - const task = pending.get(id) - pending.delete(id) - const groupKey = await groupKeyTask - task.resolve(groupKey) - })) - } catch (err) { - groupKeyIds.forEach((id) => { - if (!pending.has(id)) { return } - const task = pending.get(id) - pending.delete(id) - task.reject(err) - }) - } - } + this.pending.set(groupKeyId, Defer()) - if (!timeouts[key]) { - timeouts[key] = setTimeout(() => { - delete timeouts[key] - processBuffer() + if (!this.timeouts[key]) { + this.timeouts[key] = setTimeout(() => { + delete this.timeouts[key] + this.processBuffer({ streamId, publisherId }) }, 1000) } - return pending.get(groupKeyId) + return this.pending.get(groupKeyId) } - function cleanupPending() { - Array.from(Object.entries(timeouts)).forEach(([key, value]) => { + cleanupPending() { + Array.from(Object.entries(this.timeouts)).forEach(([key, value]) => { clearTimeout(value) - delete timeouts[key] + delete this.timeouts[key] }) - const pendingValues = Array.from(pending.values()) - pending.clear() + const pendingValues = Array.from(this.pending.values()) + this.pending.clear() pendingValues.forEach((value) => { value.resolve(undefined) }) - pMemoize.clear(getGroupKeyStore) + pMemoize.clear(this.getGroupKeyStore) } - const next = Scaffold([ - async () => { - await encryptionUtil.onReady() - }, - async () => { - sub = await SubscriberKeyExhangeSubscription(client, getGroupKeyStore, encryptionUtil) - return async () => { - if (!sub) { return } - const cancelTask = sub.cancel() - sub = undefined - await cancelTask - } - } - ], async () => enabled, { - id: `SubscriberKeyExhangeSubscription.${client.id}`, - onChange(shouldUp) { - if (!shouldUp) { - cleanupPending() + initNext() { + return Scaffold([ + async () => { + await this.encryptionUtil.onReady() + }, + async () => { + this.sub = await SubscriberKeyExhangeSubscription(this.client, this.getGroupKeyStore, this.encryptionUtil) + return async () => { + if (!this.sub) { return } + const cancelTask = this.sub.cancel() + this.sub = undefined + await cancelTask + } } - }, - async onDone() { - // clean up requestKey - if (requestKeysStep) { - await requestKeysStep() + ], async () => this.enabled, { + id: `SubscriberKeyExhangeSubscription.${this.client.id}`, + onChange: (shouldUp) => { + if (!shouldUp) { + this.cleanupPending() + } + }, + onDone: async () => { + // clean up requestKey + if (this.requestKeysStep) { + await this.requestKeysStep() + } } - } - }) + }) + } - async function getGroupKey(streamMessage: StreamMessage) { + async getGroupKey(streamMessage: StreamMessage) { if (!streamMessage.groupKeyId) { return [] } - await next() - if (!enabled) { return [] } + await this.next() + if (!this.enabled) { return [] } - return getKey(streamMessage) + return this.getKey(streamMessage) } - return Object.assign(getGroupKey, { - async start() { - enabled = true - return next() - }, - async addNewKey(streamMessage: StreamMessage) { - if (!streamMessage.newGroupKey) { return } - const streamId = streamMessage.getStreamId() - const groupKeyStore = await getGroupKeyStore(streamId) - // newGroupKey has been converted into GroupKey - const { newGroupKey } = streamMessage - await groupKeyStore.add(newGroupKey as GroupKey) - }, - async stop() { - enabled = false - return next() - } - }) + async start() { + this.enabled = true + return this.next() + } + + async addNewKey(streamMessage: StreamMessage) { + if (!streamMessage.newGroupKey) { return } + const streamId = streamMessage.getStreamId() + const groupKeyStore = await this.getGroupKeyStore(streamId) + // newGroupKey has been converted into GroupKey + const groupKey: unknown = streamMessage.newGroupKey + await groupKeyStore.add(groupKey as GroupKey) + } + + async stop() { + this.enabled = false + return this.next() + } } diff --git a/src/subscribe/Decrypt.js b/src/subscribe/Decrypt.js index c51a6d66c..934ecea74 100644 --- a/src/subscribe/Decrypt.js +++ b/src/subscribe/Decrypt.js @@ -17,7 +17,7 @@ export default function Decrypt(client, options = {}) { } } - const requestKey = SubscriberKeyExchange(client, { + const keyExchange = new SubscriberKeyExchange(client, { ...options, groupKeys: { ...client.options.groupKeys, @@ -38,7 +38,7 @@ export default function Decrypt(client, options = {}) { } try { - const groupKey = await requestKey(streamMessage).catch((err) => { + const groupKey = await keyExchange.getGroupKey(streamMessage).catch((err) => { throw new UnableToDecryptError(`Could not get GroupKey: ${streamMessage.groupKeyId} – ${err.message}`, streamMessage) }) @@ -46,7 +46,7 @@ export default function Decrypt(client, options = {}) { throw new UnableToDecryptError(`Group key not found: ${streamMessage.groupKeyId}`, streamMessage) } await EncryptionUtil.decryptStreamMessage(streamMessage, groupKey) - requestKey.addNewKey(streamMessage) + await keyExchange.addNewKey(streamMessage) } catch (err) { await onError(err, streamMessage) } finally { @@ -57,7 +57,7 @@ export default function Decrypt(client, options = {}) { return Object.assign(decrypt, { async stop() { - return requestKey.stop() + return keyExchange.stop() } }) } From c43250f980e6e99d2d99a873c5697c6ce521297c Mon Sep 17 00:00:00 2001 From: Tim Oxley Date: Thu, 6 May 2021 17:45:16 -0400 Subject: [PATCH 23/28] types: Type resolved value from Defer. --- src/utils/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/index.ts b/src/utils/index.ts index 9a637eef4..8690dfa3e 100644 --- a/src/utils/index.ts +++ b/src/utils/index.ts @@ -243,7 +243,7 @@ export function Defer(executor: (...args: Parameters['then']>) => } // eslint-disable-next-line promise/param-names - const p = new Promise((_resolve, _reject) => { + const p: Promise = new Promise((_resolve, _reject) => { resolveFn = _resolve rejectFn = _reject executor(resolve, reject) From cc23d7d7b7029b86748db573d4ec03beeb6b968d Mon Sep 17 00:00:00 2001 From: Tim Oxley Date: Thu, 6 May 2021 17:46:11 -0400 Subject: [PATCH 24/28] refactor: Simplify key exchange subscriber by removing buffering & pending promise de-duplication. --- .../encryption/KeyExchangeSubscriber.ts | 210 ++++-------------- 1 file changed, 43 insertions(+), 167 deletions(-) diff --git a/src/stream/encryption/KeyExchangeSubscriber.ts b/src/stream/encryption/KeyExchangeSubscriber.ts index 176001208..209b1ccbd 100644 --- a/src/stream/encryption/KeyExchangeSubscriber.ts +++ b/src/stream/encryption/KeyExchangeSubscriber.ts @@ -3,8 +3,6 @@ import { } from 'streamr-client-protocol' import pMemoize from 'p-memoize' import { uuid, Defer } from '../../utils' -import Scaffold from '../../utils/Scaffold' -import mem from 'mem' import { validateOptions } from '../utils' import EncryptionUtil, { GroupKey } from './Encryption' @@ -22,7 +20,7 @@ import { type MessageMatch = (content: any, streamMessage: StreamMessage) => boolean function waitForSubMessage(sub: Subscription, matchFn: MessageMatch) { - const task = Defer() + const task = Defer() const onMessage = (content: any, streamMessage: StreamMessage) => { try { if (matchFn(content, streamMessage)) { @@ -85,12 +83,7 @@ export class SubscriberKeyExchange { client initialGroupKeys encryptionUtil - pending = new Map>() - getBuffer = mem<(groupKeyId: GroupKeyId) => GroupKeyId[], [string]>(() => []) - timeouts: Record> = Object.create(null) - next enabled = true - sub?: Subscription constructor(client: StreamrClient, { groupKeys = {} }: KeyExhangeOptions = {}) { this.client = client @@ -102,7 +95,10 @@ export class SubscriberKeyExchange { } }) this.encryptionUtil = new EncryptionUtil(client.options.keyExchange) - this.next = this.initNext() + } + + async getSubscription() { + return SubscriberKeyExhangeSubscription(this.client, this.getGroupKeyStore, this.encryptionUtil) } async requestKeys({ streamId, publisherId, groupKeyIds }: { @@ -110,72 +106,40 @@ export class SubscriberKeyExchange { publisherId: string, groupKeyIds: GroupKeyId[] }) { - let done = false const requestId = uuid('GroupKeyRequest') const rsaPublicKey = this.encryptionUtil.getPublicKey() const keyExchangeStreamId = getKeyExchangeStreamId(publisherId) - let responseTask: ReturnType - let cancelTask: ReturnType - let receivedGroupKeys: GroupKey[] = [] - let response: any - - this.requestKeysStep = Scaffold([ - async () => { - if (!this.sub) { throw new Error('no subscription') } - cancelTask = Defer() - responseTask = waitForSubMessage(this.sub, (content, streamMessage) => { - const { messageType } = streamMessage - const matchesMessageType = ( - messageType === StreamMessage.MESSAGE_TYPES.GROUP_KEY_RESPONSE - || messageType === StreamMessage.MESSAGE_TYPES.GROUP_KEY_ERROR_RESPONSE - ) - - if (!matchesMessageType) { - return false - } - - const groupKeyResponse = GroupKeyResponse.fromArray(content) - return groupKeyResponse.requestId === requestId - }) - - cancelTask.then(responseTask.resolve).catch(responseTask.reject) - return () => { - cancelTask.resolve(undefined) - } - }, async () => { - const msg = new GroupKeyRequest({ - streamId, - requestId, - rsaPublicKey, - groupKeyIds, - }) - await this.client.publish(keyExchangeStreamId, msg) - }, async () => { - response = await responseTask - return () => { - response = undefined + let sub!: Subscription + try { + sub = await this.getSubscription() + const responseTask = waitForSubMessage(sub, (content, streamMessage) => { + const { messageType } = streamMessage + const matchesMessageType = ( + messageType === StreamMessage.MESSAGE_TYPES.GROUP_KEY_RESPONSE + || messageType === StreamMessage.MESSAGE_TYPES.GROUP_KEY_ERROR_RESPONSE + ) + + if (!matchesMessageType) { + return false } - }, async () => { - receivedGroupKeys = response ? await getGroupKeysFromStreamMessage(response, this.encryptionUtil) : [] - return () => { - receivedGroupKeys = [] - } - }, - ], async () => this.enabled && !done, { - id: `requestKeys.${requestId}`, - onChange(isGoingUp) { - if (!isGoingUp && cancelTask) { - cancelTask.resolve(undefined) - } + const groupKeyResponse = GroupKeyResponse.fromArray(content) + return groupKeyResponse.requestId === requestId + }) + const msg = new GroupKeyRequest({ + streamId, + requestId, + rsaPublicKey, + groupKeyIds, + }) + await this.client.publish(keyExchangeStreamId, msg) + const response = await responseTask + return response ? getGroupKeysFromStreamMessage(response, this.encryptionUtil) : [] + } finally { + if (sub) { + await sub.unsubscribe() } - }) - - await this.requestKeysStep() - const keys = receivedGroupKeys.slice() - done = true - await this.requestKeysStep() - return keys + } } async getGroupKeyStore(streamId: string) { @@ -187,46 +151,6 @@ export class SubscriberKeyExchange { }) } - private async processBuffer({ streamId, publisherId }: { streamId: string, publisherId: string }) { - if (!this.enabled) { return } - const key = `${streamId}.${publisherId}` - const currentBuffer = this.getBuffer(key) - const groupKeyIds = currentBuffer.slice() - const groupKeyStore = await this.getGroupKeyStore(streamId) - currentBuffer.length = 0 - try { - const receivedGroupKeys = await this.requestKeys({ - streamId, - publisherId, - groupKeyIds, - }) - if (!this.enabled) { return } - await Promise.all(receivedGroupKeys.map(async (groupKey) => ( - groupKeyStore.add(groupKey) - ))) - if (!this.enabled) { return } - await Promise.all(groupKeyIds.map(async (id) => { - if (!this.pending.has(id)) { return } - const groupKeyTask = groupKeyStore.get(id) - const task = this.pending.get(id) - this.pending.delete(id) - const groupKey = await groupKeyTask - if (task) { - task.resolve(groupKey) - } - })) - } catch (err) { - groupKeyIds.forEach((id) => { - if (!this.pending.has(id)) { return } - const task = this.pending.get(id) - this.pending.delete(id) - if (task) { - task.reject(err) - } - }) - } - } - async getKey(streamMessage: StreamMessage) { const streamId = streamMessage.getStreamId() const publisherId = streamMessage.getPublisherId() @@ -245,79 +169,32 @@ export class SubscriberKeyExchange { return existingGroupKey } - if (this.pending.has(groupKeyId)) { - return this.pending.get(groupKeyId) - } + const receivedGroupKeys = await this.requestKeys({ + streamId, + publisherId, + groupKeyIds: [groupKeyId], + }) - const key = `${streamId}.${publisherId}` - const buffer = this.getBuffer(key) - buffer.push(groupKeyId) - this.pending.set(groupKeyId, Defer()) + await Promise.all(receivedGroupKeys.map(async (groupKey: GroupKey) => ( + groupKeyStore.add(groupKey) + ))) - if (!this.timeouts[key]) { - this.timeouts[key] = setTimeout(() => { - delete this.timeouts[key] - this.processBuffer({ streamId, publisherId }) - }, 1000) - } - - return this.pending.get(groupKeyId) + return groupKeyStore.get(groupKeyId) } cleanupPending() { - Array.from(Object.entries(this.timeouts)).forEach(([key, value]) => { - clearTimeout(value) - delete this.timeouts[key] - }) - const pendingValues = Array.from(this.pending.values()) - this.pending.clear() - pendingValues.forEach((value) => { - value.resolve(undefined) - }) pMemoize.clear(this.getGroupKeyStore) } - initNext() { - return Scaffold([ - async () => { - await this.encryptionUtil.onReady() - }, - async () => { - this.sub = await SubscriberKeyExhangeSubscription(this.client, this.getGroupKeyStore, this.encryptionUtil) - return async () => { - if (!this.sub) { return } - const cancelTask = this.sub.cancel() - this.sub = undefined - await cancelTask - } - } - ], async () => this.enabled, { - id: `SubscriberKeyExhangeSubscription.${this.client.id}`, - onChange: (shouldUp) => { - if (!shouldUp) { - this.cleanupPending() - } - }, - onDone: async () => { - // clean up requestKey - if (this.requestKeysStep) { - await this.requestKeysStep() - } - } - }) - } - async getGroupKey(streamMessage: StreamMessage) { if (!streamMessage.groupKeyId) { return [] } - await this.next() - if (!this.enabled) { return [] } + await this.encryptionUtil.onReady() return this.getKey(streamMessage) } async start() { this.enabled = true - return this.next() } async addNewKey(streamMessage: StreamMessage) { @@ -331,6 +208,5 @@ export class SubscriberKeyExchange { async stop() { this.enabled = false - return this.next() } } From bc64e3faa8e90af75be5b8e4597d67aa0e30f625 Mon Sep 17 00:00:00 2001 From: Tim Oxley Date: Thu, 6 May 2021 21:25:25 -0400 Subject: [PATCH 25/28] refactor: Convert integration/Encryption test to typescript. --- ...{Encryption.test.js => Encryption.test.ts} | 87 ++++++++++--------- 1 file changed, 47 insertions(+), 40 deletions(-) rename test/integration/{Encryption.test.js => Encryption.test.ts} (90%) diff --git a/test/integration/Encryption.test.js b/test/integration/Encryption.test.ts similarity index 90% rename from test/integration/Encryption.test.js rename to test/integration/Encryption.test.ts index 6ad6b83df..151d5fa08 100644 --- a/test/integration/Encryption.test.js +++ b/test/integration/Encryption.test.ts @@ -5,6 +5,8 @@ import { describeRepeats, fakePrivateKey, uid, Msg, getPublishTestMessages } fro import { Defer } from '../../src/utils' import { StreamrClient } from '../../src/StreamrClient' import { GroupKey } from '../../src/stream/encryption/Encryption' +import { Stream, StreamOperation } from '../../src/stream' +import { Subscription } from '../../src' import Connection from '../../src/Connection' import { StorageNode } from '../../src/stream/StorageNode' @@ -15,17 +17,17 @@ const TIMEOUT = 10 * 1000 const { StreamMessage } = MessageLayer describeRepeats('decryption', () => { - let publishTestMessages + let publishTestMessages: ReturnType let expectErrors = 0 // check no errors by default - let errors = [] + let errors: Error[] = [] - const getOnError = (errs) => jest.fn((err) => { + const getOnError = (errs: Error[]) => jest.fn((err) => { errs.push(err) }) let onError = jest.fn() - let client - let stream + let client: StreamrClient + let stream: Stream const createClient = (opts = {}) => { const c = new StreamrClient({ @@ -35,6 +37,7 @@ describeRepeats('decryption', () => { }, autoConnect: false, autoDisconnect: false, + // @ts-expect-error disconnectDelay: 1, publishAutoDisconnectDelay: 50, maxRetries: 2, @@ -46,7 +49,7 @@ describeRepeats('decryption', () => { return c } - function checkEncryptionMessages(testClient) { + function checkEncryptionMessages(testClient: StreamrClient) { const onSendTest = Defer() testClient.connection.on('_send', onSendTest.wrapError((sendingMsg) => { // check encryption is as expected @@ -73,7 +76,7 @@ describeRepeats('decryption', () => { }) afterEach(async () => { - await wait() + await wait(0) // ensure no unexpected errors expect(errors).toHaveLength(expectErrors) if (client) { @@ -82,7 +85,7 @@ describeRepeats('decryption', () => { }) afterEach(async () => { - await wait() + await wait(0) if (client) { client.debug('disconnecting after test') await client.disconnect() @@ -95,7 +98,7 @@ describeRepeats('decryption', () => { } }) - async function setupClient(opts) { + async function setupClient(opts?: any) { client = createClient(opts) await Promise.all([ client.session.getSessionToken(), @@ -130,6 +133,7 @@ describeRepeats('decryption', () => { const done = Defer() const sub = await client.subscribe({ stream: stream.id, + // @ts-expect-error groupKeys: keys, }, done.wrap((parsedContent, streamMessage) => { expect(parsedContent).toEqual(msg) @@ -174,7 +178,7 @@ describeRepeats('decryption', () => { client.publish(stream.id, msg), done, ]) - onEncryptionMessageErr.resolve() // will be ignored if errored + onEncryptionMessageErr.resolve(undefined) // will be ignored if errored await onEncryptionMessageErr // All good, unsubscribe await client.unsubscribe(sub) @@ -191,7 +195,7 @@ describeRepeats('decryption', () => { // Check signature stuff received.push(streamMessage) if (received.length === msgs.length) { - done.resolve() + done.resolve(undefined) } })) @@ -238,7 +242,7 @@ describeRepeats('decryption', () => { } }) - onEncryptionMessageErr.resolve() // will be ignored if errored + onEncryptionMessageErr.resolve(undefined) // will be ignored if errored await onEncryptionMessageErr // All good, unsubscribe await client.unsubscribe(sub) @@ -246,6 +250,7 @@ describeRepeats('decryption', () => { it('errors if rotating group key for no stream', async () => { expect(async () => ( + // @ts-expect-error client.rotateGroupKey() )).rejects.toThrow('streamId') }) @@ -257,8 +262,8 @@ describeRepeats('decryption', () => { }) it('allows other users to get group key', async () => { - let otherClient - let sub + let otherClient: StreamrClient + let sub: Subscription try { otherClient = createClient({ autoConnect: true, @@ -268,8 +273,8 @@ describeRepeats('decryption', () => { const onEncryptionMessageErr = checkEncryptionMessages(client) const onEncryptionMessageErr2 = checkEncryptionMessages(otherClient) const otherUser = await otherClient.getUserInfo() - await stream.grantPermission('stream_get', otherUser.username) - await stream.grantPermission('stream_subscribe', otherUser.username) + await stream.grantPermission(StreamOperation.STREAM_GET, otherUser.username) + await stream.grantPermission(StreamOperation.STREAM_SUBSCRIBE, otherUser.username) const done = Defer() const msg = Msg() @@ -293,9 +298,9 @@ describeRepeats('decryption', () => { client.publish(stream.id, msg), done, ]) - onEncryptionMessageErr.resolve() // will be ignored if errored + onEncryptionMessageErr.resolve(undefined) // will be ignored if errored await onEncryptionMessageErr - onEncryptionMessageErr2.resolve() // will be ignored if errored + onEncryptionMessageErr2.resolve(undefined) // will be ignored if errored await onEncryptionMessageErr2 } finally { if (otherClient) { @@ -317,7 +322,7 @@ describeRepeats('decryption', () => { let didFindStream2 = false - function checkEncryptionMessagesPerStream(testClient) { + function checkEncryptionMessagesPerStream(testClient: StreamrClient) { const onSendTest = Defer() testClient.connection.on('_send', onSendTest.wrapError((sendingMsg) => { // check encryption is as expected @@ -344,7 +349,7 @@ describeRepeats('decryption', () => { return onSendTest } - async function testSub(testStream) { + async function testSub(testStream: Stream) { const NUM_MESSAGES = 5 const done = Defer() const received = [] @@ -353,7 +358,7 @@ describeRepeats('decryption', () => { }, done.wrapError((parsedContent) => { received.push(parsedContent) if (received.length === NUM_MESSAGES) { - done.resolve() + done.resolve(undefined) } })) sub.once('error', done.reject) @@ -376,7 +381,7 @@ describeRepeats('decryption', () => { testSub(stream), testSub(stream2), ]) - onEncryptionMessageErr.resolve() // will be ignored if errored + onEncryptionMessageErr.resolve(undefined) // will be ignored if errored await onEncryptionMessageErr expect(didFindStream2).toBeTruthy() }, TIMEOUT) @@ -393,7 +398,7 @@ describeRepeats('decryption', () => { const groupKey2 = GroupKey.generate() await client.setNextGroupKey(stream2.id, groupKey2) - function checkEncryptionMessagesPerStream(testClient) { + function checkEncryptionMessagesPerStream(testClient: StreamrClient) { const onSendTest = Defer() testClient.connection.on('_send', onSendTest.wrapError((sendingMsg) => { // check encryption is as expected @@ -421,7 +426,7 @@ describeRepeats('decryption', () => { return onSendTest } - async function testSub(testStream) { + async function testSub(testStream: Stream) { const NUM_MESSAGES = 5 const done = Defer() const received = [] @@ -430,7 +435,7 @@ describeRepeats('decryption', () => { }, done.wrapError((parsedContent) => { received.push(parsedContent) if (received.length === NUM_MESSAGES) { - done.resolve() + done.resolve(undefined) } })) sub.once('error', done.reject) @@ -450,7 +455,7 @@ describeRepeats('decryption', () => { testSub(stream), testSub(stream2), ]) - onEncryptionMessageErr.resolve() // will be ignored if errored + onEncryptionMessageErr.resolve(undefined) // will be ignored if errored await onEncryptionMessageErr }, TIMEOUT) @@ -586,6 +591,7 @@ describeRepeats('decryption', () => { const sub = await client.subscribe({ stream: stream.id, + // @ts-expect-error groupKeys: keys, }) @@ -618,7 +624,7 @@ describeRepeats('decryption', () => { }) describe('revoking permissions', () => { - let client2 + let client2: StreamrClient beforeEach(async () => { client2 = createClient({ id: 'subscriber' }) @@ -643,8 +649,8 @@ describeRepeats('decryption', () => { const MAX_MESSAGES = 6 await client.rotateGroupKey(stream.id) - const p1 = await stream.grantPermission('stream_get', client2.getPublisherId()) - const p2 = await stream.grantPermission('stream_subscribe', client2.getPublisherId()) + const p1 = await stream.grantPermission(StreamOperation.STREAM_GET, await client2.getPublisherId()) + const p2 = await stream.grantPermission(StreamOperation.STREAM_SUBSCRIBE, await client2.getPublisherId()) const sub = await client2.subscribe({ stream: stream.id, @@ -676,12 +682,13 @@ describeRepeats('decryption', () => { } } }) - let t + + let t!: ReturnType await expect(async () => { for await (const m of sub) { received.push(m.getParsedContent()) if (received.length === REVOKE_AFTER) { - gotMessages.resolve() + gotMessages.resolve(undefined) clearTimeout(t) t = setTimeout(() => { sub.cancel() @@ -704,7 +711,7 @@ describeRepeats('decryption', () => { expect(onSubError).toHaveBeenCalledTimes(1) }) - it('fails gracefully if permission revoked with low cache maxAge fail first message', async () => { + it.only('fails gracefully if permission revoked with low cache maxAge fail first message', async () => { await client.disconnect() await setupClient({ id: 'publisher', @@ -715,8 +722,8 @@ describeRepeats('decryption', () => { const MAX_MESSAGES = 3 await client.rotateGroupKey(stream.id) - const p1 = await stream.grantPermission('stream_get', client2.getPublisherId()) - const p2 = await stream.grantPermission('stream_subscribe', client2.getPublisherId()) + const p1 = await stream.grantPermission(StreamOperation.STREAM_GET, await client2.getPublisherId()) + const p2 = await stream.grantPermission(StreamOperation.STREAM_SUBSCRIBE, await client2.getPublisherId()) const sub = await client2.subscribe({ stream: stream.id, @@ -749,12 +756,12 @@ describeRepeats('decryption', () => { } } }) - let t + let t!: ReturnType await expect(async () => { for await (const m of sub) { received.push(m.getParsedContent()) if (received.length === REVOKE_AFTER) { - gotMessages.resolve() + gotMessages.resolve(undefined) clearTimeout(t) t = setTimeout(() => { sub.cancel() @@ -788,8 +795,8 @@ describeRepeats('decryption', () => { const MAX_MESSAGES = 10 await client.rotateGroupKey(stream.id) - const p1 = await stream.grantPermission('stream_get', client2.getPublisherId()) - const p2 = await stream.grantPermission('stream_subscribe', client2.getPublisherId()) + const p1 = await stream.grantPermission(StreamOperation.STREAM_GET, await client2.getPublisherId()) + const p2 = await stream.grantPermission(StreamOperation.STREAM_SUBSCRIBE, await client2.getPublisherId()) const sub = await client2.subscribe({ stream: stream.id, @@ -822,12 +829,12 @@ describeRepeats('decryption', () => { } }) - let t + let t!: ReturnType await expect(async () => { for await (const m of sub) { received.push(m.getParsedContent()) if (received.length === REVOKE_AFTER) { - gotMessages.resolve() + gotMessages.resolve(undefined) clearTimeout(t) t = setTimeout(() => { sub.cancel() From d9d1771a5e36a72dc595ebe8d2bcf20133a2bd8b Mon Sep 17 00:00:00 2001 From: Tim Oxley Date: Fri, 7 May 2021 15:01:47 -0400 Subject: [PATCH 26/28] fix: Increase encryption test robustness & improve decryption failure error messages. --- src/subscribe/Decrypt.js | 10 +- test/integration/Encryption.test.ts | 1110 +++++++++++++-------------- 2 files changed, 525 insertions(+), 595 deletions(-) diff --git a/src/subscribe/Decrypt.js b/src/subscribe/Decrypt.js index 934ecea74..6c77d653d 100644 --- a/src/subscribe/Decrypt.js +++ b/src/subscribe/Decrypt.js @@ -10,7 +10,7 @@ export default function Decrypt(client, options = {}) { // noop unless message encrypted return (streamMessage) => { if (streamMessage.groupKeyId) { - throw new Error('No keyExchange configured, cannot decrypt message.') + throw new UnableToDecryptError('No keyExchange configured, cannot decrypt any message.', streamMessage) } return streamMessage @@ -43,11 +43,17 @@ export default function Decrypt(client, options = {}) { }) if (!groupKey) { - throw new UnableToDecryptError(`Group key not found: ${streamMessage.groupKeyId}`, streamMessage) + throw new UnableToDecryptError([ + `Could not get GroupKey: ${streamMessage.groupKeyId}`, + 'Publisher is offline, key does not exist or no permission to access key.', + ].join(' '), streamMessage) } + await EncryptionUtil.decryptStreamMessage(streamMessage, groupKey) await keyExchange.addNewKey(streamMessage) } catch (err) { + // clear cached permissions if cannot decrypt, likely permissions need updating + client.cached.clearStream(streamMessage.getStreamId()) await onError(err, streamMessage) } finally { yield streamMessage diff --git a/test/integration/Encryption.test.ts b/test/integration/Encryption.test.ts index 151d5fa08..97a918e3d 100644 --- a/test/integration/Encryption.test.ts +++ b/test/integration/Encryption.test.ts @@ -6,12 +6,13 @@ import { Defer } from '../../src/utils' import { StreamrClient } from '../../src/StreamrClient' import { GroupKey } from '../../src/stream/encryption/Encryption' import { Stream, StreamOperation } from '../../src/stream' -import { Subscription } from '../../src' import Connection from '../../src/Connection' import { StorageNode } from '../../src/stream/StorageNode' +import Debug from 'debug' import config from './config' +const debug = Debug('StreamrClient::test') const TIMEOUT = 10 * 1000 const { StreamMessage } = MessageLayer @@ -26,7 +27,8 @@ describeRepeats('decryption', () => { }) let onError = jest.fn() - let client: StreamrClient + let publisher: StreamrClient + let subscriber: StreamrClient let stream: Stream const createClient = (opts = {}) => { @@ -79,17 +81,24 @@ describeRepeats('decryption', () => { await wait(0) // ensure no unexpected errors expect(errors).toHaveLength(expectErrors) - if (client) { - expect(client.onError).toHaveBeenCalledTimes(expectErrors) + if (publisher) { + expect(publisher.onError).toHaveBeenCalledTimes(expectErrors) } }) - afterEach(async () => { + async function cleanupClient(client?: StreamrClient, msg = 'disconnecting after test') { await wait(0) if (client) { - client.debug('disconnecting after test') + client.debug(msg) await client.disconnect() } + } + + afterEach(async () => { + await Promise.all([ + cleanupClient(publisher), + cleanupClient(subscriber), + ]) const openSockets = Connection.getOpen() if (openSockets !== 0) { @@ -99,187 +108,91 @@ describeRepeats('decryption', () => { }) async function setupClient(opts?: any) { - client = createClient(opts) + const client = createClient(opts) await Promise.all([ client.session.getSessionToken(), client.connect(), ]) + return client + } + async function setupStream() { const name = uid('stream') - stream = await client.createStream({ + stream = await publisher.createStream({ name, requireEncryptedData: true, }) await stream.addToStorageNode(StorageNode.STREAMR_DOCKER_DEV) - publishTestMessages = getPublishTestMessages(client, { + publishTestMessages = getPublishTestMessages(publisher, { stream }) } - beforeEach(async () => { - await setupClient() - }) - - it('client.subscribe can decrypt encrypted messages if it knows the group key', async () => { - const groupKey = GroupKey.generate() - const keys = { - [stream.id]: { - [groupKey.id]: groupKey, - } - } - const msg = Msg() - const done = Defer() - const sub = await client.subscribe({ - stream: stream.id, - // @ts-expect-error - groupKeys: keys, - }, done.wrap((parsedContent, streamMessage) => { - expect(parsedContent).toEqual(msg) - // Check signature stuff - expect(streamMessage.signatureType).toBe(StreamMessage.SIGNATURE_TYPES.ETH) - expect(streamMessage.getPublisherId()) - expect(streamMessage.signature) - })) - - client.once('error', done.reject) - await client.setNextGroupKey(stream.id, groupKey) - // Publish after subscribed - await Promise.all([ - client.publish(stream.id, msg), - done, + async function setupPublisherSubscriberClients(opts?: any) { + debug('set up clients', opts) + // eslint-disable-next-line require-atomic-updates, semi-style + ;[publisher, subscriber] = await Promise.all([ + setupClient({ id: uid('publisher'), ...opts }), + setupClient({ id: uid('subscriber'), ...opts }), ]) - // All good, unsubscribe - await client.unsubscribe(sub) - }) - - it('client.subscribe can get the group key and decrypt encrypted message using an RSA key pair', async () => { - const done = Defer() - const msg = Msg() - const groupKey = GroupKey.generate() - // subscribe without knowing the group key to decrypt stream messages - const sub = await client.subscribe({ - stream: stream.id, - }, done.wrap((parsedContent, streamMessage) => { - expect(parsedContent).toEqual(msg) - - // Check signature stuff - expect(streamMessage.signatureType).toBe(StreamMessage.SIGNATURE_TYPES.ETH) - expect(streamMessage.getPublisherId()) - expect(streamMessage.signature) - })) - sub.once('error', done.reject) + } - await client.setNextGroupKey(stream.id, groupKey) - const onEncryptionMessageErr = checkEncryptionMessages(client) + async function grantSubscriberPermissions({ stream: s = stream, client: c = subscriber }: { stream?: Stream, client?: StreamrClient } = {}) { + const p1 = await s.grantPermission(StreamOperation.STREAM_GET, await c.getPublisherId()) + const p2 = await s.grantPermission(StreamOperation.STREAM_SUBSCRIBE, await c.getPublisherId()) + return [p1, p2] + } - await Promise.all([ - client.publish(stream.id, msg), - done, - ]) - onEncryptionMessageErr.resolve(undefined) // will be ignored if errored - await onEncryptionMessageErr - // All good, unsubscribe - await client.unsubscribe(sub) - }, TIMEOUT) - - it('changing group key injects group key into next stream message', async () => { - const done = Defer() - const msgs = [Msg(), Msg(), Msg()] - const received = [] - // subscribe without knowing the group key to decrypt stream messages - const sub = await client.subscribe({ - stream: stream.id, - }, done.wrapError((_parsedContent, streamMessage) => { - // Check signature stuff - received.push(streamMessage) - if (received.length === msgs.length) { - done.resolve(undefined) - } - })) + describe('using default config', () => { + beforeEach(async () => { + await setupPublisherSubscriberClients() + }) + beforeEach(async () => { + await setupStream() + }) - sub.once('error', done.reject) - - const onEncryptionMessageErr = checkEncryptionMessages(client) - // id | groupKeyId | newGroupKey (encrypted by groupKeyId) - // msg1 gk2 - - // msg2 gk2 gk3 - // msg3 gk3 - - const groupKey1 = GroupKey.generate() - const groupKey2 = GroupKey.generate() - await client.setNextGroupKey(stream.id, groupKey1) - await client.publish(stream.id, msgs[0]) - await client.setNextGroupKey(stream.id, groupKey2) - await client.publish(stream.id, msgs[1]) - await client.publish(stream.id, msgs[2]) - await done - expect(received.map((m) => m.getParsedContent())).toEqual(msgs) - received.forEach((streamMessage, index) => { - expect(streamMessage.signatureType).toBe(StreamMessage.SIGNATURE_TYPES.ETH) - expect(streamMessage.getPublisherId()) - expect(streamMessage.signature) - switch (index) { - case 0: { - expect(streamMessage.newGroupKey).toEqual(null) - expect(streamMessage.groupKeyId).toEqual(groupKey1.id) - break - } - case 1: { - expect(streamMessage.newGroupKey).toEqual(groupKey2) - expect(streamMessage.groupKeyId).toEqual(groupKey1.id) - break - } - case 2: { - expect(streamMessage.newGroupKey).toEqual(null) - expect(streamMessage.groupKeyId).toEqual(groupKey2.id) - break - } - default: { - throw new Error(`should not get here: ${index}`) + it('client.subscribe can decrypt encrypted messages if it knows the group key', async () => { + const groupKey = GroupKey.generate() + const keys = { + [stream.id]: { + [groupKey.id]: groupKey, } - } - }) - - onEncryptionMessageErr.resolve(undefined) // will be ignored if errored - await onEncryptionMessageErr - // All good, unsubscribe - await client.unsubscribe(sub) - }, TIMEOUT) - - it('errors if rotating group key for no stream', async () => { - expect(async () => ( - // @ts-expect-error - client.rotateGroupKey() - )).rejects.toThrow('streamId') - }) - - it('errors if setting group key for no stream', async () => { - expect(async () => ( - client.setNextGroupKey(undefined, GroupKey.generate()) - )).rejects.toThrow('streamId') - }) - - it('allows other users to get group key', async () => { - let otherClient: StreamrClient - let sub: Subscription - try { - otherClient = createClient({ - autoConnect: true, - autoDisconnect: true, - }) + const msg = Msg() + const done = Defer() + await grantSubscriberPermissions() + const sub = await subscriber.subscribe({ + stream: stream.id, + // @ts-expect-error + groupKeys: keys, + }, done.wrap((parsedContent, streamMessage) => { + expect(parsedContent).toEqual(msg) + // Check signature stuff + expect(streamMessage.signatureType).toBe(StreamMessage.SIGNATURE_TYPES.ETH) + expect(streamMessage.getPublisherId()) + expect(streamMessage.signature) + })) - const onEncryptionMessageErr = checkEncryptionMessages(client) - const onEncryptionMessageErr2 = checkEncryptionMessages(otherClient) - const otherUser = await otherClient.getUserInfo() - await stream.grantPermission(StreamOperation.STREAM_GET, otherUser.username) - await stream.grantPermission(StreamOperation.STREAM_SUBSCRIBE, otherUser.username) + publisher.once('error', done.reject) + await publisher.setNextGroupKey(stream.id, groupKey) + // Publish after subscribed + await Promise.all([ + publisher.publish(stream.id, msg), + done, + ]) + // All good, unsubscribe + await publisher.unsubscribe(sub) + }) + it('client.subscribe can get the group key and decrypt encrypted message using an RSA key pair', async () => { const done = Defer() const msg = Msg() + const groupKey = GroupKey.generate() // subscribe without knowing the group key to decrypt stream messages - sub = await otherClient.subscribe({ + await grantSubscriberPermissions() + const sub = await subscriber.subscribe({ stream: stream.id, }, done.wrap((parsedContent, streamMessage) => { expect(parsedContent).toEqual(msg) @@ -291,521 +204,482 @@ describeRepeats('decryption', () => { })) sub.once('error', done.reject) - const groupKey = GroupKey.generate() - await client.setNextGroupKey(stream.id, groupKey) + await publisher.setNextGroupKey(stream.id, groupKey) + const onEncryptionMessageErr = checkEncryptionMessages(publisher) await Promise.all([ - client.publish(stream.id, msg), + publisher.publish(stream.id, msg), done, ]) onEncryptionMessageErr.resolve(undefined) // will be ignored if errored await onEncryptionMessageErr - onEncryptionMessageErr2.resolve(undefined) // will be ignored if errored - await onEncryptionMessageErr2 - } finally { - if (otherClient) { - if (sub) { - await otherClient.unsubscribe(sub) - } - await otherClient.disconnect() - await otherClient.logout() - } - } - }, TIMEOUT) - - it('does not encrypt messages in stream without groupkey', async () => { - const name = uid('stream') - const stream2 = await client.createStream({ - name, - requireEncryptedData: false, - }) - - let didFindStream2 = false - - function checkEncryptionMessagesPerStream(testClient: StreamrClient) { - const onSendTest = Defer() - testClient.connection.on('_send', onSendTest.wrapError((sendingMsg) => { - // check encryption is as expected - const { streamMessage } = sendingMsg - if (!streamMessage) { - return - } - - if (streamMessage.getStreamId() === stream2.id) { - didFindStream2 = true - // stream2 always unencrypted - expect(streamMessage.encryptionType).toEqual(StreamMessage.ENCRYPTION_TYPES.NONE) - return - } + // All good, unsubscribe + await publisher.unsubscribe(sub) + }, TIMEOUT) - if (streamMessage.messageType === StreamMessage.MESSAGE_TYPES.MESSAGE) { - expect(streamMessage.encryptionType).toEqual(StreamMessage.ENCRYPTION_TYPES.AES) - } else if (streamMessage.messageType === StreamMessage.MESSAGE_TYPES.GROUP_KEY_RESPONSE) { - expect(streamMessage.encryptionType).toEqual(StreamMessage.ENCRYPTION_TYPES.RSA) - } else { - expect(streamMessage.encryptionType).toEqual(StreamMessage.ENCRYPTION_TYPES.NONE) - } - })) - return onSendTest - } - - async function testSub(testStream: Stream) { - const NUM_MESSAGES = 5 + it('changing group key injects group key into next stream message', async () => { const done = Defer() + const msgs = [Msg(), Msg(), Msg()] const received = [] - const sub = await client.subscribe({ - stream: testStream.id, - }, done.wrapError((parsedContent) => { - received.push(parsedContent) - if (received.length === NUM_MESSAGES) { + await grantSubscriberPermissions() + // subscribe without knowing the group key to decrypt stream messages + const sub = await subscriber.subscribe({ + stream: stream.id, + }, done.wrapError((_parsedContent, streamMessage) => { + // Check signature stuff + received.push(streamMessage) + if (received.length === msgs.length) { done.resolve(undefined) } })) - sub.once('error', done.reject) - const published = await publishTestMessages(NUM_MESSAGES, { - stream: testStream, - }) + sub.once('error', done.reject) + const onEncryptionMessageErr = checkEncryptionMessages(publisher) + // id | groupKeyId | newGroupKey (encrypted by groupKeyId) + // msg1 gk2 - + // msg2 gk2 gk3 + // msg3 gk3 - + const groupKey1 = GroupKey.generate() + const groupKey2 = GroupKey.generate() + await publisher.setNextGroupKey(stream.id, groupKey1) + await publisher.publish(stream.id, msgs[0]) + await publisher.setNextGroupKey(stream.id, groupKey2) + await publisher.publish(stream.id, msgs[1]) + await publisher.publish(stream.id, msgs[2]) await done + expect(received.map((m) => m.getParsedContent())).toEqual(msgs) + received.forEach((streamMessage, index) => { + expect(streamMessage.signatureType).toBe(StreamMessage.SIGNATURE_TYPES.ETH) + expect(streamMessage.getPublisherId()) + expect(streamMessage.signature) + switch (index) { + case 0: { + expect(streamMessage.newGroupKey).toEqual(null) + expect(streamMessage.groupKeyId).toEqual(groupKey1.id) + break + } + case 1: { + expect(streamMessage.newGroupKey).toEqual(groupKey2) + expect(streamMessage.groupKeyId).toEqual(groupKey1.id) + break + } + case 2: { + expect(streamMessage.newGroupKey).toEqual(null) + expect(streamMessage.groupKeyId).toEqual(groupKey2.id) + break + } + default: { + throw new Error(`should not get here: ${index}`) + } - expect(received).toEqual(published) - } - - const onEncryptionMessageErr = checkEncryptionMessagesPerStream(client) - - const groupKey = GroupKey.generate() - await client.setNextGroupKey(stream.id, groupKey) - - await Promise.all([ - testSub(stream), - testSub(stream2), - ]) - onEncryptionMessageErr.resolve(undefined) // will be ignored if errored - await onEncryptionMessageErr - expect(didFindStream2).toBeTruthy() - }, TIMEOUT) - - it('sets group key per-stream', async () => { - const name = uid('stream') - const stream2 = await client.createStream({ - name, - requireEncryptedData: true, - }) - - const groupKey = GroupKey.generate() - await client.setNextGroupKey(stream.id, groupKey) - const groupKey2 = GroupKey.generate() - await client.setNextGroupKey(stream2.id, groupKey2) - - function checkEncryptionMessagesPerStream(testClient: StreamrClient) { - const onSendTest = Defer() - testClient.connection.on('_send', onSendTest.wrapError((sendingMsg) => { - // check encryption is as expected - const { streamMessage } = sendingMsg - if (!streamMessage) { - return - } - - if (streamMessage.getStreamId() === stream2.id) { - expect(streamMessage.groupKeyId).toEqual(groupKey2.id) } + }) - if (streamMessage.getStreamId() === stream.id) { - expect(streamMessage.groupKeyId).toEqual(groupKey.id) - } + onEncryptionMessageErr.resolve(undefined) // will be ignored if errored + await onEncryptionMessageErr + // All good, unsubscribe + await publisher.unsubscribe(sub) + }, TIMEOUT) + + it('errors if rotating group key for no stream', async () => { + expect(async () => ( + // @ts-expect-error + publisher.rotateGroupKey() + )).rejects.toThrow('streamId') + }) - if (streamMessage.messageType === StreamMessage.MESSAGE_TYPES.MESSAGE) { - expect(streamMessage.encryptionType).toEqual(StreamMessage.ENCRYPTION_TYPES.AES) - } else if (streamMessage.messageType === StreamMessage.MESSAGE_TYPES.GROUP_KEY_RESPONSE) { - expect(streamMessage.encryptionType).toEqual(StreamMessage.ENCRYPTION_TYPES.RSA) - } else { - expect(streamMessage.encryptionType).toEqual(StreamMessage.ENCRYPTION_TYPES.NONE) - } - })) - return onSendTest - } + it('errors if setting group key for no stream', async () => { + expect(async () => ( + publisher.setNextGroupKey(undefined, GroupKey.generate()) + )).rejects.toThrow('streamId') + }) - async function testSub(testStream: Stream) { - const NUM_MESSAGES = 5 + it('allows other users to get group key', async () => { + const onEncryptionMessageErr = checkEncryptionMessages(publisher) + const onEncryptionMessageErr2 = checkEncryptionMessages(subscriber) const done = Defer() - const received = [] - const sub = await client.subscribe({ - stream: testStream.id, - }, done.wrapError((parsedContent) => { - received.push(parsedContent) - if (received.length === NUM_MESSAGES) { - done.resolve(undefined) - } + const msg = Msg() + await grantSubscriberPermissions() + // subscribe without knowing the group key to decrypt stream messages + const sub = await subscriber.subscribe({ + stream: stream.id, + }, done.wrap((parsedContent, streamMessage) => { + expect(parsedContent).toEqual(msg) + + // Check signature stuff + expect(streamMessage.signatureType).toBe(StreamMessage.SIGNATURE_TYPES.ETH) + expect(streamMessage.getPublisherId()) + expect(streamMessage.signature) })) sub.once('error', done.reject) - const published = await publishTestMessages(NUM_MESSAGES, { - stream: testStream, - }) + const groupKey = GroupKey.generate() + await publisher.setNextGroupKey(stream.id, groupKey) - await done + await Promise.all([ + publisher.publish(stream.id, msg), + done, + ]) + onEncryptionMessageErr.resolve(undefined) // will be ignored if errored + await onEncryptionMessageErr + onEncryptionMessageErr2.resolve(undefined) // will be ignored if errored + await onEncryptionMessageErr2 + }, TIMEOUT) - expect(received).toEqual(published) - } + it('does not encrypt messages in stream without groupkey', async () => { + const name = uid('stream') + const stream2 = await publisher.createStream({ + name, + requireEncryptedData: false, + }) - const onEncryptionMessageErr = checkEncryptionMessagesPerStream(client) + let didFindStream2 = false - await Promise.all([ - testSub(stream), - testSub(stream2), - ]) - onEncryptionMessageErr.resolve(undefined) // will be ignored if errored - await onEncryptionMessageErr - }, TIMEOUT) - - it('client.subscribe can get the group key and decrypt multiple encrypted messages using an RSA key pair', async () => { - // subscribe without knowing the group key to decrypt stream messages - const sub = await client.subscribe({ - stream: stream.id, - }) + function checkEncryptionMessagesPerStream(testClient: StreamrClient) { + const onSendTest = Defer() + testClient.connection.on('_send', onSendTest.wrapError((sendingMsg) => { + // check encryption is as expected + const { streamMessage } = sendingMsg + if (!streamMessage) { + return + } - const groupKey = GroupKey.generate() - await client.setNextGroupKey(stream.id, groupKey) - const published = await publishTestMessages() + if (streamMessage.getStreamId() === stream2.id) { + didFindStream2 = true + // stream2 always unencrypted + expect(streamMessage.encryptionType).toEqual(StreamMessage.ENCRYPTION_TYPES.NONE) + return + } - const received = [] - for await (const msg of sub) { - received.push(msg.getParsedContent()) - if (received.length === published.length) { - return + if (streamMessage.messageType === StreamMessage.MESSAGE_TYPES.MESSAGE) { + expect(streamMessage.encryptionType).toEqual(StreamMessage.ENCRYPTION_TYPES.AES) + } else if (streamMessage.messageType === StreamMessage.MESSAGE_TYPES.GROUP_KEY_RESPONSE) { + expect(streamMessage.encryptionType).toEqual(StreamMessage.ENCRYPTION_TYPES.RSA) + } else { + expect(streamMessage.encryptionType).toEqual(StreamMessage.ENCRYPTION_TYPES.NONE) + } + })) + return onSendTest } - } - expect(received).toEqual(published) + async function testSub(testStream: Stream) { + const NUM_MESSAGES = 5 + const done = Defer() + const received = [] + await grantSubscriberPermissions({ stream: testStream }) + const sub = await subscriber.subscribe({ + stream: testStream.id, + }, done.wrapError((parsedContent) => { + received.push(parsedContent) + if (received.length === NUM_MESSAGES) { + done.resolve(undefined) + } + })) + sub.once('error', done.reject) - // All good, unsubscribe - await client.unsubscribe(sub) - }, TIMEOUT) + const published = await publishTestMessages(NUM_MESSAGES, { + stream: testStream, + }) - it('subscribe with changing group key', async () => { - // subscribe without knowing the group key to decrypt stream messages - const sub = await client.subscribe({ - stream: stream.id, - }) - - const published = await publishTestMessages(5, { - async beforeEach() { - await client.rotateGroupKey(stream.id) - } - }) + await done - const received = [] - for await (const msg of sub) { - received.push(msg.getParsedContent()) - if (received.length === published.length) { - return + expect(received).toEqual(published) } - } - - expect(received).toEqual(published) - - // All good, unsubscribe - await client.unsubscribe(sub) - }, TIMEOUT) - it('client.resend last can get the historical keys for previous encrypted messages', async () => { - // Publish encrypted messages with different keys - const published = await publishTestMessages(5, { - waitForLast: true, - async beforeEach() { - await client.rotateGroupKey(stream.id) - } - }) + const onEncryptionMessageErr = checkEncryptionMessagesPerStream(publisher) - // resend without knowing the historical keys - const sub = await client.resend({ - stream: stream.id, - resend: { - last: 2, - }, - }) + const groupKey = GroupKey.generate() + await publisher.setNextGroupKey(stream.id, groupKey) - const received = await sub.collect() + await Promise.all([ + testSub(stream), + testSub(stream2), + ]) + onEncryptionMessageErr.resolve(undefined) // will be ignored if errored + await onEncryptionMessageErr + expect(didFindStream2).toBeTruthy() + }, TIMEOUT) + + it('sets group key per-stream', async () => { + const name = uid('stream') + const stream2 = await publisher.createStream({ + name, + requireEncryptedData: true, + }) - expect(received).toEqual(published.slice(-2)) - await client.unsubscribe(sub) - }, TIMEOUT) + const groupKey = GroupKey.generate() + await publisher.setNextGroupKey(stream.id, groupKey) + const groupKey2 = GroupKey.generate() + await publisher.setNextGroupKey(stream2.id, groupKey2) + + function checkEncryptionMessagesPerStream(testClient: StreamrClient) { + const onSendTest = Defer() + testClient.connection.on('_send', onSendTest.wrapError((sendingMsg) => { + // check encryption is as expected + const { streamMessage } = sendingMsg + if (!streamMessage) { + return + } - it('client.subscribe with resend last can get the historical keys for previous encrypted messages', async () => { - // Publish encrypted messages with different keys - const published = await publishTestMessages(5, { - waitForLast: true, - async beforeEach() { - await client.rotateGroupKey(stream.id) - } - }) + if (streamMessage.getStreamId() === stream2.id) { + expect(streamMessage.groupKeyId).toEqual(groupKey2.id) + } - // subscribe with resend without knowing the historical keys - const sub = await client.subscribe({ - stream: stream.id, - resend: { - last: 2, - }, - }) + if (streamMessage.getStreamId() === stream.id) { + expect(streamMessage.groupKeyId).toEqual(groupKey.id) + } - const received = [] - for await (const msg of sub) { - received.push(msg.getParsedContent()) - if (received.length === 2) { - break + if (streamMessage.messageType === StreamMessage.MESSAGE_TYPES.MESSAGE) { + expect(streamMessage.encryptionType).toEqual(StreamMessage.ENCRYPTION_TYPES.AES) + } else if (streamMessage.messageType === StreamMessage.MESSAGE_TYPES.GROUP_KEY_RESPONSE) { + expect(streamMessage.encryptionType).toEqual(StreamMessage.ENCRYPTION_TYPES.RSA) + } else { + expect(streamMessage.encryptionType).toEqual(StreamMessage.ENCRYPTION_TYPES.NONE) + } + })) + return onSendTest } - } - expect(received).toEqual(published.slice(-2)) - await client.unsubscribe(sub) - }, TIMEOUT) + async function testSub(testStream: Stream) { + const NUM_MESSAGES = 5 + const done = Defer() + const received = [] + await grantSubscriberPermissions({ stream: testStream }) + const sub = await subscriber.subscribe({ + stream: testStream.id, + }, done.wrapError((parsedContent) => { + received.push(parsedContent) + if (received.length === NUM_MESSAGES) { + done.resolve(undefined) + } + })) + sub.once('error', done.reject) - it('fails gracefully if cannot decrypt', async () => { - const MAX_MESSAGES = 10 - const groupKey = GroupKey.generate() - const keys = { - [stream.id]: { - [groupKey.id]: groupKey, - } - } + const published = await publishTestMessages(NUM_MESSAGES, { + stream: testStream, + }) - await client.setNextGroupKey(stream.id, groupKey) + await done - const BAD_INDEX = 6 - let count = 0 - const { parse } = client.connection - client.connection.parse = (...args) => { - const msg = parse.call(client.connection, ...args) - if (!msg.streamMessage) { - return msg + expect(received).toEqual(published) } - if (count === BAD_INDEX) { - msg.streamMessage.groupKeyId = 'badgroupkey' - } + const onEncryptionMessageErr = checkEncryptionMessagesPerStream(publisher) - count += 1 - return msg - } + await Promise.all([ + testSub(stream), + testSub(stream2), + ]) + onEncryptionMessageErr.resolve(undefined) // will be ignored if errored + await onEncryptionMessageErr + }, TIMEOUT) - const sub = await client.subscribe({ - stream: stream.id, - // @ts-expect-error - groupKeys: keys, - }) + it('client.subscribe can get the group key and decrypt multiple encrypted messages using an RSA key pair', async () => { + // subscribe without knowing publisher the group key to decrypt stream messages + await grantSubscriberPermissions() + const sub = await subscriber.subscribe({ + stream: stream.id, + }) - const onSubError = jest.fn((err) => { - expect(err).toBeInstanceOf(Error) - expect(err.message).toMatch('decrypt') - }) + const groupKey = GroupKey.generate() + await publisher.setNextGroupKey(stream.id, groupKey) + const published = await publishTestMessages() - sub.on('error', onSubError) + const received = [] + for await (const msg of sub) { + received.push(msg.getParsedContent()) + if (received.length === published.length) { + return + } + } - // Publish after subscribed - const published = await publishTestMessages(MAX_MESSAGES, { - timestamp: 1111111, - }) + expect(received).toEqual(published) - const received = [] - for await (const m of sub) { - received.push(m.getParsedContent()) - if (received.length === published.length - 1) { - break - } - } + // All good, unsubscribe + await subscriber.unsubscribe(sub) + }, TIMEOUT) - expect(received).toEqual([ - ...published.slice(0, BAD_INDEX), - ...published.slice(BAD_INDEX + 1, MAX_MESSAGES) - ]) + it('subscribe with changing group key', async () => { + // subscribe without knowing the group key to decrypt stream messages + await grantSubscriberPermissions() + const sub = await subscriber.subscribe({ + stream: stream.id, + }) - expect(onSubError).toHaveBeenCalledTimes(1) - }) + const published = await publishTestMessages(5, { + async beforeEach() { + await publisher.rotateGroupKey(stream.id) + } + }) - describe('revoking permissions', () => { - let client2: StreamrClient + const received = [] + for await (const msg of sub) { + received.push(msg.getParsedContent()) + if (received.length === published.length) { + return + } + } - beforeEach(async () => { - client2 = createClient({ id: 'subscriber' }) - await client2.connect() - await client2.session.getSessionToken() - client2.debug('new SUBSCRIBER') - }) + expect(received).toEqual(published) - afterEach(async () => { - await client2.disconnect() - }) + // All good, unsubscribe + await publisher.unsubscribe(sub) + }, TIMEOUT) - it('fails gracefully if permission revoked with low cache maxAge', async () => { - await client.disconnect() - await setupClient({ - id: 'publisher', - cache: { - maxAge: 1, + it('client.resend last can get the historical keys for previous encrypted messages', async () => { + // Publish encrypted messages with different keys + const published = await publishTestMessages(5, { + waitForLast: true, + async beforeEach() { + await publisher.rotateGroupKey(stream.id) } }) - const MAX_MESSAGES = 6 - await client.rotateGroupKey(stream.id) - - const p1 = await stream.grantPermission(StreamOperation.STREAM_GET, await client2.getPublisherId()) - const p2 = await stream.grantPermission(StreamOperation.STREAM_SUBSCRIBE, await client2.getPublisherId()) - - const sub = await client2.subscribe({ + // resend without knowing the historical keys + await grantSubscriberPermissions() + const sub = await subscriber.resend({ stream: stream.id, + resend: { + last: 2, + }, }) - const errs = [] - const onSubError = jest.fn((err) => { - errs.push(err) - throw err + const received = await sub.collect() + + expect(received).toEqual(published.slice(-2)) + await publisher.unsubscribe(sub) + }, TIMEOUT) + + it('client.subscribe with resend last can get the historical keys for previous encrypted messages', async () => { + // Publish encrypted messages with different keys + const published = await publishTestMessages(5, { + waitForLast: true, + async beforeEach() { + await publisher.rotateGroupKey(stream.id) + } }) - sub.on('error', onSubError) + await grantSubscriberPermissions() + // subscribe with resend without knowing the historical keys + const sub = await subscriber.subscribe({ + stream: stream.id, + resend: { + last: 2, + }, + }) const received = [] - // Publish after subscribed - let count = 0 - const REVOKE_AFTER = 3 - const gotMessages = Defer() - // do publish in background otherwise permission is revoked before subscriber starts processing - const publishTask = publishTestMessages(MAX_MESSAGES, { - timestamp: 1111111, - async afterEach() { - count += 1 - if (count === REVOKE_AFTER) { - await gotMessages - await stream.revokePermission(p1.id) - await stream.revokePermission(p2.id) - await client.rekey(stream.id) - } + for await (const msg of sub) { + received.push(msg.getParsedContent()) + if (received.length === 2) { + break } - }) + } - let t!: ReturnType - await expect(async () => { - for await (const m of sub) { - received.push(m.getParsedContent()) - if (received.length === REVOKE_AFTER) { - gotMessages.resolve(undefined) - clearTimeout(t) - t = setTimeout(() => { - sub.cancel() - }, 10000) - } + expect(received).toEqual(published.slice(-2)) + await subscriber.unsubscribe(sub) + }, TIMEOUT) - if (received.length === MAX_MESSAGES) { - clearTimeout(t) - break - } + it('fails gracefully if cannot decrypt', async () => { + const MAX_MESSAGES = 10 + const groupKey = GroupKey.generate() + const keys = { + [stream.id]: { + [groupKey.id]: groupKey, } - }).rejects.toThrow('not a subscriber') - clearTimeout(t) - const published = await publishTask + } - expect(received).toEqual([ - ...published.slice(0, REVOKE_AFTER), - ]) + await publisher.setNextGroupKey(stream.id, groupKey) - expect(onSubError).toHaveBeenCalledTimes(1) - }) + const BAD_INDEX = 6 + let count = 0 + const { parse } = subscriber.connection + subscriber.connection.parse = (...args) => { + const msg = parse.call(subscriber.connection, ...args) + if (!msg.streamMessage) { + return msg + } - it.only('fails gracefully if permission revoked with low cache maxAge fail first message', async () => { - await client.disconnect() - await setupClient({ - id: 'publisher', - cache: { - maxAge: 1, + if (count === BAD_INDEX) { + msg.streamMessage.groupKeyId = 'badgroupkey' } - }) - const MAX_MESSAGES = 3 - await client.rotateGroupKey(stream.id) - const p1 = await stream.grantPermission(StreamOperation.STREAM_GET, await client2.getPublisherId()) - const p2 = await stream.grantPermission(StreamOperation.STREAM_SUBSCRIBE, await client2.getPublisherId()) + count += 1 + return msg + } - const sub = await client2.subscribe({ + await grantSubscriberPermissions() + const sub = await subscriber.subscribe({ stream: stream.id, + // @ts-expect-error + groupKeys: keys, }) - sub.debug('sub', sub.id) - const errs = [] const onSubError = jest.fn((err) => { - errs.push(err) - throw err + expect(err).toBeInstanceOf(Error) + expect(err.message).toMatch('decrypt') }) sub.on('error', onSubError) - const received = [] // Publish after subscribed - let count = 0 - const REVOKE_AFTER = 1 - const gotMessages = Defer() - // do publish in background otherwise permission is revoked before subscriber starts processing - const publishTask = publishTestMessages(MAX_MESSAGES, { + const published = await publishTestMessages(MAX_MESSAGES, { timestamp: 1111111, - async afterEach() { - count += 1 - if (count === REVOKE_AFTER) { - await gotMessages - await stream.revokePermission(p1.id) - await stream.revokePermission(p2.id) - await client.rekey(stream.id) - } - } }) - let t!: ReturnType - await expect(async () => { - for await (const m of sub) { - received.push(m.getParsedContent()) - if (received.length === REVOKE_AFTER) { - gotMessages.resolve(undefined) - clearTimeout(t) - t = setTimeout(() => { - sub.cancel() - }, 10000) - } - if (received.length === MAX_MESSAGES) { - clearTimeout(t) - break - } + const received = [] + for await (const m of sub) { + received.push(m.getParsedContent()) + if (received.length === published.length - 1) { + break } - }).rejects.toThrow('not a subscriber') - clearTimeout(t) - const published = await publishTask + } expect(received).toEqual([ - ...published.slice(0, REVOKE_AFTER), + ...published.slice(0, BAD_INDEX), + ...published.slice(BAD_INDEX + 1, MAX_MESSAGES) ]) expect(onSubError).toHaveBeenCalledTimes(1) }) + }) - it('fails gracefully if permission revoked with high cache maxAge', async () => { - await client.disconnect() - await setupClient({ - cache: { - maxAge: 99999999, - } - }) - - const MAX_MESSAGES = 10 - await client.rotateGroupKey(stream.id) - - const p1 = await stream.grantPermission(StreamOperation.STREAM_GET, await client2.getPublisherId()) - const p2 = await stream.grantPermission(StreamOperation.STREAM_SUBSCRIBE, await client2.getPublisherId()) - - const sub = await client2.subscribe({ + describe('revoking permissions', () => { + async function testRevokeDuringSubscribe({ + maxMessages = 6, + revokeAfter = Math.round(maxMessages / 2), + }: { + maxMessages?: number + revokeAfter?: number + } = {}) { + // this has publisher & subscriber clients + // publisher begins publishing `maxMessages` messages + // subscriber recieves messages + // after publisher publishes `revokeAfter` messages, + // and subscriber receives the last message + // subscriber has subscribe permission removed + // and publisher rekeys the stream. + // Publisher then keep publishing messages with the new key. + // The subscriber should error on the next message, and unsubscribe + // due to permission change. + // check that subscriber only got messages from before permission revoked + // and subscriber errored with something about group key or + // permissions + + await publisher.rotateGroupKey(stream.id) + + await stream.grantPermission(StreamOperation.STREAM_GET, await subscriber.getPublisherId()) + const subPermission = await stream.grantPermission(StreamOperation.STREAM_SUBSCRIBE, await subscriber.getPublisherId()) + + const sub = await subscriber.subscribe({ stream: stream.id, }) const errs = [] const onSubError = jest.fn((err) => { errs.push(err) - throw err + throw err // this should trigger unsub }) sub.on('error', onSubError) @@ -813,48 +687,98 @@ describeRepeats('decryption', () => { const received = [] // Publish after subscribed let count = 0 - const REVOKE_AFTER = 6 const gotMessages = Defer() // do publish in background otherwise permission is revoked before subscriber starts processing - const publishTask = publishTestMessages(MAX_MESSAGES, { + const publishTask = publishTestMessages(maxMessages, { timestamp: 1111111, async afterEach() { count += 1 - if (count === REVOKE_AFTER) { + publisher.debug('PUBLISHED %d of %d', count, maxMessages) + if (count === revokeAfter) { await gotMessages - await stream.revokePermission(p1.id) - await stream.revokePermission(p2.id) - await client.rekey(stream.id) + await stream.revokePermission(subPermission.id) + await publisher.rekey(stream.id) } } }) + subscriber.debug('\n\n1\n\n') let t!: ReturnType - await expect(async () => { - for await (const m of sub) { - received.push(m.getParsedContent()) - if (received.length === REVOKE_AFTER) { - gotMessages.resolve(undefined) - clearTimeout(t) - t = setTimeout(() => { - sub.cancel() - }, 2000) + const timedOut = jest.fn() + try { + await expect(async () => { + for await (const m of sub) { + subscriber.debug('got', m.getParsedContent()) + received.push(m.getParsedContent()) + if (received.length === revokeAfter) { + gotMessages.resolve(undefined) + clearTimeout(t) + t = setTimeout(() => { + timedOut() + sub.cancel() + }, 6000) + } + + if (received.length === maxMessages) { + clearTimeout(t) + break + } } + }).rejects.toThrow(/not a subscriber|Could not get GroupKey/) + } finally { + clearTimeout(t) + // run in finally to ensure publish promise finishes before + // continuing no matter the result of the expect call above + const published = await publishTask + + expect(received).toEqual([ + ...published.slice(0, revokeAfter), + ]) + + expect(onSubError).toHaveBeenCalledTimes(1) + expect(timedOut).toHaveBeenCalledTimes(0) + } + } - if (received.length === MAX_MESSAGES) { - clearTimeout(t) - break + describe('low cache maxAge', () => { + beforeEach(async () => { + await setupPublisherSubscriberClients({ + cache: { + maxAge: 1, } - } - }).rejects.toThrow('decrypt') - clearTimeout(t) - const published = await publishTask + }) + }) + beforeEach(async () => { + await setupStream() + }) + it('fails gracefully if permission revoked after first message', async () => { + await testRevokeDuringSubscribe({ maxMessages: 6, revokeAfter: 1 }) + }) + it('fails gracefully if permission revoked after some messages', async () => { + await testRevokeDuringSubscribe({ maxMessages: 6, revokeAfter: 3 }) + }) + }) - expect(received).toEqual([ - ...published.slice(0, REVOKE_AFTER), - ]) + describe('high cache maxAge', () => { + beforeEach(async () => { + await setupPublisherSubscriberClients({ + cache: { + maxAge: 9999999, + } + }) + }) - expect(onSubError).toHaveBeenCalledTimes(1) + beforeEach(async () => { + await setupStream() + }) + + it('fails gracefully if permission revoked after first message', async () => { + await testRevokeDuringSubscribe({ maxMessages: 6, revokeAfter: 1 }) + }) + + it('fails gracefully if permission revoked after some messages', async () => { + await testRevokeDuringSubscribe({ maxMessages: 6, revokeAfter: 3 }) + }) }) }) }) From 13f3d4ed7efe72854e5e078e280e1374dee8714f Mon Sep 17 00:00:00 2001 From: Tim Oxley Date: Mon, 10 May 2021 16:32:16 -0400 Subject: [PATCH 27/28] types: Fix Encryption test types. --- test/integration/Encryption.test.ts | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/test/integration/Encryption.test.ts b/test/integration/Encryption.test.ts index 97a918e3d..9a147b255 100644 --- a/test/integration/Encryption.test.ts +++ b/test/integration/Encryption.test.ts @@ -220,7 +220,7 @@ describeRepeats('decryption', () => { it('changing group key injects group key into next stream message', async () => { const done = Defer() const msgs = [Msg(), Msg(), Msg()] - const received = [] + const received: any[] = [] await grantSubscriberPermissions() // subscribe without knowing the group key to decrypt stream messages const sub = await subscriber.subscribe({ @@ -291,6 +291,7 @@ describeRepeats('decryption', () => { it('errors if setting group key for no stream', async () => { expect(async () => ( + // @ts-expect-error publisher.setNextGroupKey(undefined, GroupKey.generate()) )).rejects.toThrow('streamId') }) @@ -366,7 +367,7 @@ describeRepeats('decryption', () => { async function testSub(testStream: Stream) { const NUM_MESSAGES = 5 const done = Defer() - const received = [] + const received: any = [] await grantSubscriberPermissions({ stream: testStream }) const sub = await subscriber.subscribe({ stream: testStream.id, @@ -444,7 +445,7 @@ describeRepeats('decryption', () => { async function testSub(testStream: Stream) { const NUM_MESSAGES = 5 const done = Defer() - const received = [] + const received: any[] = [] await grantSubscriberPermissions({ stream: testStream }) const sub = await subscriber.subscribe({ stream: testStream.id, @@ -597,11 +598,12 @@ describeRepeats('decryption', () => { const { parse } = subscriber.connection subscriber.connection.parse = (...args) => { const msg = parse.call(subscriber.connection, ...args) - if (!msg.streamMessage) { + if (!('streamMessage' in msg)) { return msg } if (count === BAD_INDEX) { + // @ts-expect-error msg.streamMessage.groupKeyId = 'badgroupkey' } @@ -676,15 +678,15 @@ describeRepeats('decryption', () => { stream: stream.id, }) - const errs = [] - const onSubError = jest.fn((err) => { + const errs: Error[] = [] + const onSubError = jest.fn((err: Error) => { errs.push(err) throw err // this should trigger unsub }) sub.on('error', onSubError) - const received = [] + const received: any[] = [] // Publish after subscribed let count = 0 const gotMessages = Defer() From 3b58f505df02ff00bfb41795a5540022e69a07bb Mon Sep 17 00:00:00 2001 From: Tim Oxley Date: Tue, 11 May 2021 12:41:01 -0400 Subject: [PATCH 28/28] test: Fix EncryptionKeyPersistence test name. --- test/integration/EncryptionKeyPersistence.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/EncryptionKeyPersistence.test.ts b/test/integration/EncryptionKeyPersistence.test.ts index 4f8caf199..e0815b3bc 100644 --- a/test/integration/EncryptionKeyPersistence.test.ts +++ b/test/integration/EncryptionKeyPersistence.test.ts @@ -10,7 +10,7 @@ import config from './config' const TIMEOUT = 10 * 1000 -describeRepeats('decryption', () => { +describeRepeats('Encryption Key Persistence', () => { let expectErrors = 0 // check no errors by default let errors: Error[] = [] let onError = jest.fn()