From 75a99cf39ad434e017855ae93fdd6db64693833c Mon Sep 17 00:00:00 2001 From: Murod Khaydarov Date: Sun, 25 Oct 2020 22:43:34 +0300 Subject: [PATCH 01/12] save changes --- workers/grouper/src/index.ts | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/workers/grouper/src/index.ts b/workers/grouper/src/index.ts index 993bb21c..f9a7aa25 100644 --- a/workers/grouper/src/index.ts +++ b/workers/grouper/src/index.ts @@ -10,6 +10,7 @@ import { GroupedEventDBScheme, RepetitionDBScheme } from 'hawk.types'; import { DatabaseReadWriteError, ValidationError } from '../../../lib/workerErrors'; import { decodeUnsafeFields, encodeUnsafeFields } from '../../../lib/utils/unsafeFields'; import HawkCatcher from '@hawk.so/nodejs'; +import CacheManager from '../../../lib/cache/controller'; /** * Error code of MongoDB key duplication error @@ -167,11 +168,14 @@ export default class GrouperWorker extends Worker { if (isUserFromOriginalEvent) { return false; } else { - const repetition = await this.db.getConnection().collection(`repetitions:${task.projectId}`) - .findOne({ - groupHash: existedEvent.groupHash, - 'payload.user.id': eventUser.id, - }); + const repetitionCacheKey = `repetitions:${task.projectId}:${existedEvent.groupHash}:${eventUser.id}`; + const repetition = CacheManager.get(repetitionCacheKey, () => { + return this.db.getConnection().collection(`repetitions:${task.projectId}`) + .findOne({ + groupHash: existedEvent.groupHash, + 'payload.user.id': eventUser.id, + }); + }) /** * If there is no repetitions from this user — return true @@ -191,13 +195,15 @@ export default class GrouperWorker extends Worker { throw new ValidationError('Controller.saveEvent: Project ID is invalid or missed'); } - try { + const eventCacheKey = `${projectId}:${query.toString()}` + return CacheManager.get(eventCacheKey, () => { return this.db.getConnection() .collection(`events:${projectId}`) - .findOne(query); - } catch (err) { - throw new DatabaseReadWriteError(err); - } + .findOne(query) + .catch((err) => { + throw new DatabaseReadWriteError(err); + }) + }); } /** From c18cf21562d98e72a4f58f152ca0a9b3db799e14 Mon Sep 17 00:00:00 2001 From: Murod Khaydarov Date: Sun, 25 Oct 2020 22:57:35 +0300 Subject: [PATCH 02/12] memoize hashing --- workers/grouper/src/index.ts | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/workers/grouper/src/index.ts b/workers/grouper/src/index.ts index f9a7aa25..c0c7d160 100644 --- a/workers/grouper/src/index.ts +++ b/workers/grouper/src/index.ts @@ -31,15 +31,27 @@ export default class GrouperWorker extends Worker { */ private db: DatabaseController = new DatabaseController(process.env.MONGO_EVENTS_DATABASE_URI); + /** + * Memoized Hashing computation + */ + private static cachedHashValues: {[key: string]: string} = {}; + /** * Get unique hash from event data * * @param task - worker task to create hash */ private static getUniqueEventHash(task: GroupWorkerTask): string { - return crypto.createHmac('sha256', process.env.EVENT_SECRET) - .update(task.catcherType + task.event.title) - .digest('hex'); + const computedHashValueCacheKey = `${task.catcherType}:${task.event.title}`; + + if (!this.cachedHashValues[computedHashValueCacheKey]) { + this.cachedHashValues[computedHashValueCacheKey] = crypto.createHmac('sha256', process.env.EVENT_SECRET) + .update(task.catcherType + task.event.title) + .digest('hex'); + } + + return this.cachedHashValues[computedHashValueCacheKey]; + } /** From c6ab8415bcc73cd708f0d3bac3b8235e8f7534d9 Mon Sep 17 00:00:00 2001 From: Murod Khaydarov Date: Tue, 10 Nov 2020 15:14:40 +0300 Subject: [PATCH 03/12] update --- workers/grouper/src/index.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/workers/grouper/src/index.ts b/workers/grouper/src/index.ts index c0c7d160..8c7fa9c1 100644 --- a/workers/grouper/src/index.ts +++ b/workers/grouper/src/index.ts @@ -10,7 +10,6 @@ import { GroupedEventDBScheme, RepetitionDBScheme } from 'hawk.types'; import { DatabaseReadWriteError, ValidationError } from '../../../lib/workerErrors'; import { decodeUnsafeFields, encodeUnsafeFields } from '../../../lib/utils/unsafeFields'; import HawkCatcher from '@hawk.so/nodejs'; -import CacheManager from '../../../lib/cache/controller'; /** * Error code of MongoDB key duplication error @@ -181,7 +180,7 @@ export default class GrouperWorker extends Worker { return false; } else { const repetitionCacheKey = `repetitions:${task.projectId}:${existedEvent.groupHash}:${eventUser.id}`; - const repetition = CacheManager.get(repetitionCacheKey, () => { + const repetition = this.cache.get(repetitionCacheKey, () => { return this.db.getConnection().collection(`repetitions:${task.projectId}`) .findOne({ groupHash: existedEvent.groupHash, @@ -208,7 +207,7 @@ export default class GrouperWorker extends Worker { } const eventCacheKey = `${projectId}:${query.toString()}` - return CacheManager.get(eventCacheKey, () => { + return this.cache.get(eventCacheKey, () => { return this.db.getConnection() .collection(`events:${projectId}`) .findOne(query) From 138f487fef3721068218991636148d255b3983d1 Mon Sep 17 00:00:00 2001 From: Murod Khaydarov Date: Tue, 10 Nov 2020 15:19:01 +0300 Subject: [PATCH 04/12] init cache --- workers/grouper/src/index.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/workers/grouper/src/index.ts b/workers/grouper/src/index.ts index 8c7fa9c1..868b7bf5 100644 --- a/workers/grouper/src/index.ts +++ b/workers/grouper/src/index.ts @@ -58,6 +58,7 @@ export default class GrouperWorker extends Worker { */ public async start(): Promise { await this.db.connect(); + this.prepareCache(); await super.start(); } From 457fb166b20847945b1f2695dd4966be8a834f96 Mon Sep 17 00:00:00 2001 From: ilyamore88 Date: Wed, 9 Dec 2020 21:56:28 +0300 Subject: [PATCH 05/12] fix eslint --- workers/grouper/src/index.ts | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/workers/grouper/src/index.ts b/workers/grouper/src/index.ts index 4279719d..0f253402 100644 --- a/workers/grouper/src/index.ts +++ b/workers/grouper/src/index.ts @@ -27,14 +27,14 @@ export default class GrouperWorker extends Worker { public readonly type: string = pkg.workerType; /** - * Database Controller + * Memoized Hashing computation */ - private db: DatabaseController = new DatabaseController(process.env.MONGO_EVENTS_DATABASE_URI); + private static cachedHashValues: {[key: string]: string} = {}; /** - * Memoized Hashing computation + * Database Controller */ - private static cachedHashValues: {[key: string]: string} = {}; + private db: DatabaseController = new DatabaseController(process.env.MONGO_EVENTS_DATABASE_URI); /** * Get unique hash from event data @@ -51,7 +51,6 @@ export default class GrouperWorker extends Worker { } return this.cachedHashValues[computedHashValueCacheKey]; - } /** @@ -188,7 +187,7 @@ export default class GrouperWorker extends Worker { groupHash: existedEvent.groupHash, 'payload.user.id': eventUser.id, }); - }) + }); /** * If there is no repetitions from this user — return true @@ -208,14 +207,15 @@ export default class GrouperWorker extends Worker { throw new ValidationError('Controller.saveEvent: Project ID is invalid or missed'); } - const eventCacheKey = `${projectId}:${query.toString()}` + const eventCacheKey = `${projectId}:${query.toString()}`; + return this.cache.get(eventCacheKey, () => { return this.db.getConnection() .collection(`events:${projectId}`) .findOne(query) .catch((err) => { throw new DatabaseReadWriteError(err); - }) + }); }); } @@ -297,7 +297,7 @@ export default class GrouperWorker extends Worker { } /** - * saves event at the special aggregation collection + * Saves event at the special aggregation collection * * @param {string} projectId - project's identifier * @param {string} eventHash - event hash From 81b98b7b34430e965dde54d2fbda158ecc1baa75 Mon Sep 17 00:00:00 2001 From: ilyamore88 Date: Wed, 9 Dec 2020 22:50:09 +0300 Subject: [PATCH 06/12] add missing await and async --- workers/grouper/src/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workers/grouper/src/index.ts b/workers/grouper/src/index.ts index 0f253402..f5354036 100644 --- a/workers/grouper/src/index.ts +++ b/workers/grouper/src/index.ts @@ -181,7 +181,7 @@ export default class GrouperWorker extends Worker { return false; } else { const repetitionCacheKey = `repetitions:${task.projectId}:${existedEvent.groupHash}:${eventUser.id}`; - const repetition = this.cache.get(repetitionCacheKey, () => { + const repetition = await this.cache.get(repetitionCacheKey, async () => { return this.db.getConnection().collection(`repetitions:${task.projectId}`) .findOne({ groupHash: existedEvent.groupHash, From b1acf32e652a9675f64e9ed9989e224fe636a693 Mon Sep 17 00:00:00 2001 From: ilyamore88 Date: Wed, 9 Dec 2020 22:51:14 +0300 Subject: [PATCH 07/12] add missing async --- workers/grouper/src/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workers/grouper/src/index.ts b/workers/grouper/src/index.ts index f5354036..d37f5f36 100644 --- a/workers/grouper/src/index.ts +++ b/workers/grouper/src/index.ts @@ -209,7 +209,7 @@ export default class GrouperWorker extends Worker { const eventCacheKey = `${projectId}:${query.toString()}`; - return this.cache.get(eventCacheKey, () => { + return this.cache.get(eventCacheKey, async () => { return this.db.getConnection() .collection(`events:${projectId}`) .findOne(query) From 2379f7af7d31fe13fb712884424ce7df1d88fc6c Mon Sep 17 00:00:00 2001 From: ilyamore88 Date: Mon, 14 Dec 2020 22:23:54 +0300 Subject: [PATCH 08/12] add clearCache method --- workers/grouper/src/index.ts | 7 +++++++ workers/grouper/tests/index.test.ts | 4 ++++ 2 files changed, 11 insertions(+) diff --git a/workers/grouper/src/index.ts b/workers/grouper/src/index.ts index d37f5f36..425244f9 100644 --- a/workers/grouper/src/index.ts +++ b/workers/grouper/src/index.ts @@ -163,6 +163,13 @@ export default class GrouperWorker extends Worker { } } + /** + * Forced clears worker cache + */ + public clearCache(): void { + this.cache.flushAll(); + } + /** * Decides whether to increase the number of affected users. * diff --git a/workers/grouper/tests/index.test.ts b/workers/grouper/tests/index.test.ts index 5b58e88e..e1d44618 100644 --- a/workers/grouper/tests/index.test.ts +++ b/workers/grouper/tests/index.test.ts @@ -91,7 +91,11 @@ describe('GrouperWorker', () => { repetitionsCollection = connection.db().collection('repetitions:' + testGroupingTask.projectId); }); + /** + * Clears worker cache and mongodb before each test + */ beforeEach(async () => { + worker.clearCache(); await eventsCollection.deleteMany({}); await dailyEventsCollection.deleteMany({}); await repetitionsCollection.deleteMany({}); From 8ce48404b8235b78004ff5ba7321dea2806f23d2 Mon Sep 17 00:00:00 2001 From: ilyamore88 Date: Tue, 15 Dec 2020 20:25:23 +0300 Subject: [PATCH 09/12] add type to query field, use JSON.stringify --- workers/grouper/src/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/workers/grouper/src/index.ts b/workers/grouper/src/index.ts index 425244f9..308d2bfe 100644 --- a/workers/grouper/src/index.ts +++ b/workers/grouper/src/index.ts @@ -209,12 +209,12 @@ export default class GrouperWorker extends Worker { * @param projectId - project's identifier * @param query - mongo query string */ - private async getEvent(projectId: string, query): Promise { + private async getEvent(projectId: string, query: Record): Promise { if (!mongodb.ObjectID.isValid(projectId)) { throw new ValidationError('Controller.saveEvent: Project ID is invalid or missed'); } - const eventCacheKey = `${projectId}:${query.toString()}`; + const eventCacheKey = `${projectId}:${JSON.stringify(query)}`; return this.cache.get(eventCacheKey, async () => { return this.db.getConnection() From fb8a84ec340427f2c071bda25fcf88ed47914796 Mon Sep 17 00:00:00 2001 From: ilyamore88 Date: Tue, 15 Dec 2020 20:34:32 +0300 Subject: [PATCH 10/12] rewrite doc --- workers/grouper/src/index.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/workers/grouper/src/index.ts b/workers/grouper/src/index.ts index 308d2bfe..2278d31c 100644 --- a/workers/grouper/src/index.ts +++ b/workers/grouper/src/index.ts @@ -27,9 +27,14 @@ export default class GrouperWorker extends Worker { public readonly type: string = pkg.workerType; /** - * Memoized Hashing computation + * Contains hashed events by his catcher type and event title as keys + * + * @example + * { + * 'grouper:Hawk client catcher test': '7e2b961c35b915dcbe2704e144e8d2c3517e2c5281a5de4403c0c58978b435a0' + * } */ - private static cachedHashValues: {[key: string]: string} = {}; + private static cachedHashValues: Record = {}; /** * Database Controller From 62f19c910c7895089fbaab9d744dc77b971d782a Mon Sep 17 00:00:00 2001 From: ilyamore88 Date: Tue, 23 Feb 2021 18:09:13 +0300 Subject: [PATCH 11/12] fix doc --- workers/grouper/src/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workers/grouper/src/index.ts b/workers/grouper/src/index.ts index 2278d31c..21388801 100644 --- a/workers/grouper/src/index.ts +++ b/workers/grouper/src/index.ts @@ -27,7 +27,7 @@ export default class GrouperWorker extends Worker { public readonly type: string = pkg.workerType; /** - * Contains hashed events by his catcher type and event title as keys + * Contains event grouphashes by its catcher type and event title as keys * * @example * { From cb5e1ef5133a27a90c1fc35325b9418e7104bd23 Mon Sep 17 00:00:00 2001 From: ilyamore88 Date: Tue, 23 Feb 2021 18:26:55 +0300 Subject: [PATCH 12/12] move clearCache to abstract worker --- lib/worker.ts | 7 +++++++ workers/grouper/src/index.ts | 7 ------- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/worker.ts b/lib/worker.ts index 15438ebe..b07f2111 100644 --- a/lib/worker.ts +++ b/lib/worker.ts @@ -173,6 +173,13 @@ export abstract class Worker { ); } + /** + * Forced clears worker cache + */ + public clearCache(): void { + this.cache.flushAll(); + } + /** * Create cache controller instance */ diff --git a/workers/grouper/src/index.ts b/workers/grouper/src/index.ts index 21388801..b071555b 100644 --- a/workers/grouper/src/index.ts +++ b/workers/grouper/src/index.ts @@ -168,13 +168,6 @@ export default class GrouperWorker extends Worker { } } - /** - * Forced clears worker cache - */ - public clearCache(): void { - this.cache.flushAll(); - } - /** * Decides whether to increase the number of affected users. *