From 8a271b21515d5e62bfd350fd99a1877870868a98 Mon Sep 17 00:00:00 2001 From: Basit Chonka Date: Thu, 31 Oct 2024 09:46:52 +0100 Subject: [PATCH 1/4] use semaphore --- .../compass-user-data/src/semaphore.spec.ts | 28 +++++++++++++++++++ packages/compass-user-data/src/semaphore.ts | 27 ++++++++++++++++++ packages/compass-user-data/src/user-data.ts | 4 +++ 3 files changed, 59 insertions(+) create mode 100644 packages/compass-user-data/src/semaphore.spec.ts create mode 100644 packages/compass-user-data/src/semaphore.ts diff --git a/packages/compass-user-data/src/semaphore.spec.ts b/packages/compass-user-data/src/semaphore.spec.ts new file mode 100644 index 00000000000..3eb90550ef8 --- /dev/null +++ b/packages/compass-user-data/src/semaphore.spec.ts @@ -0,0 +1,28 @@ +import { expect } from 'chai'; +import { Semaphore } from './semaphore'; + +describe('semaphore', function () { + const maxConcurrentOps = 5; + let semaphore: Semaphore; + let taskHandler: (id: number) => Promise; + + beforeEach(() => { + semaphore = new Semaphore(maxConcurrentOps); + taskHandler = async (id: number) => { + const release = await semaphore.waitForRelease(); + const delay = Math.floor(Math.random() * 450) + 50; + try { + await new Promise((resolve) => setTimeout(resolve, delay)); + return id; + } finally { + release(); + } + }; + }); + + it('should run operations concurrently', async function () { + const tasks = Array.from({ length: 10 }, (_, i) => taskHandler(i)); + const results = await Promise.all(tasks); + expect(results).to.have.lengthOf(10); + }); +}); diff --git a/packages/compass-user-data/src/semaphore.ts b/packages/compass-user-data/src/semaphore.ts new file mode 100644 index 00000000000..98c8a8c047b --- /dev/null +++ b/packages/compass-user-data/src/semaphore.ts @@ -0,0 +1,27 @@ +export class Semaphore { + private currentCount = 0; + private queue: (() => void)[] = []; + constructor(private maxConcurrentOps: number) {} + + waitForRelease(): Promise<() => void> { + return new Promise((resolve) => { + const attempt = () => { + this.currentCount++; + resolve(this.release.bind(this)); + }; + if (this.currentCount < this.maxConcurrentOps) { + attempt(); + } else { + this.queue.push(attempt); + } + }); + } + + private release() { + this.currentCount--; + if (this.queue.length > 0) { + const next = this.queue.shift(); + next && next(); + } + } +} diff --git a/packages/compass-user-data/src/user-data.ts b/packages/compass-user-data/src/user-data.ts index bbb01daa8ef..b71c1412233 100644 --- a/packages/compass-user-data/src/user-data.ts +++ b/packages/compass-user-data/src/user-data.ts @@ -4,6 +4,7 @@ import { createLogger } from '@mongodb-js/compass-logging'; import { getStoragePath } from '@mongodb-js/compass-utils'; import type { z } from 'zod'; import writeFile from 'write-file-atomic'; +import { Semaphore } from './semaphore'; const { log, mongoLogId } = createLogger('COMPASS-USER-STORAGE'); @@ -68,6 +69,7 @@ export class UserData { private readonly serialize: SerializeContent>; private readonly deserialize: DeserializeContent; private readonly getFileName: GetFileName; + private readonly semaphore = new Semaphore(1000); constructor( private readonly validator: T, @@ -122,6 +124,7 @@ export class UserData { let data: string; let stats: Stats; let handle: fs.FileHandle | undefined = undefined; + const release = await this.semaphore.waitForRelease(); try { handle = await fs.open(absolutePath, 'r'); [stats, data] = await Promise.all([ @@ -139,6 +142,7 @@ export class UserData { throw error; } finally { await handle?.close(); + release(); } try { From d2e9cbad66f04e6b6f576d16ea7deb5e33a1660b Mon Sep 17 00:00:00 2001 From: Basit Chonka Date: Thu, 31 Oct 2024 09:54:52 +0100 Subject: [PATCH 2/4] many files test --- packages/compass-user-data/src/user-data.spec.ts | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/packages/compass-user-data/src/user-data.spec.ts b/packages/compass-user-data/src/user-data.spec.ts index e1f2ead78cb..829b5b9ccd3 100644 --- a/packages/compass-user-data/src/user-data.spec.ts +++ b/packages/compass-user-data/src/user-data.spec.ts @@ -158,6 +158,20 @@ describe('user-data', function () { expect(mongoshData?.[1]).to.be.instanceOf(Stats); } }); + + it('reads many number of files', async function () { + const files = Array.from({ length: 10000 }, (_, i) => [ + `data${i}.json`, + JSON.stringify({ name: `VSCode${i}` }), + ]); + + await Promise.all( + files.map(([filepath, data]) => writeFileToStorage(filepath, data)) + ); + + const result = await getUserData().readAll(); + expect(result.data).to.have.lengthOf(10000); + }); }); context('UserData.readOne', function () { From 01b2f94697b6e7758e2ab2fba86222c769047397 Mon Sep 17 00:00:00 2001 From: Basit Chonka Date: Thu, 31 Oct 2024 09:59:27 +0100 Subject: [PATCH 3/4] handle error --- packages/compass-user-data/src/user-data.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/compass-user-data/src/user-data.ts b/packages/compass-user-data/src/user-data.ts index b71c1412233..768de9e8e89 100644 --- a/packages/compass-user-data/src/user-data.ts +++ b/packages/compass-user-data/src/user-data.ts @@ -124,8 +124,9 @@ export class UserData { let data: string; let stats: Stats; let handle: fs.FileHandle | undefined = undefined; - const release = await this.semaphore.waitForRelease(); + let release: (() => void) | undefined = undefined; try { + release = await this.semaphore.waitForRelease(); handle = await fs.open(absolutePath, 'r'); [stats, data] = await Promise.all([ handle.stat(), @@ -142,7 +143,7 @@ export class UserData { throw error; } finally { await handle?.close(); - release(); + release?.(); } try { From fa93b5dd9817022878384e8e4256c881858f8dee Mon Sep 17 00:00:00 2001 From: Basit Chonka Date: Fri, 1 Nov 2024 12:04:12 +0100 Subject: [PATCH 4/4] cr fixes --- packages/compass-user-data/src/user-data.spec.ts | 14 -------------- packages/compass-user-data/src/user-data.ts | 2 +- 2 files changed, 1 insertion(+), 15 deletions(-) diff --git a/packages/compass-user-data/src/user-data.spec.ts b/packages/compass-user-data/src/user-data.spec.ts index 829b5b9ccd3..e1f2ead78cb 100644 --- a/packages/compass-user-data/src/user-data.spec.ts +++ b/packages/compass-user-data/src/user-data.spec.ts @@ -158,20 +158,6 @@ describe('user-data', function () { expect(mongoshData?.[1]).to.be.instanceOf(Stats); } }); - - it('reads many number of files', async function () { - const files = Array.from({ length: 10000 }, (_, i) => [ - `data${i}.json`, - JSON.stringify({ name: `VSCode${i}` }), - ]); - - await Promise.all( - files.map(([filepath, data]) => writeFileToStorage(filepath, data)) - ); - - const result = await getUserData().readAll(); - expect(result.data).to.have.lengthOf(10000); - }); }); context('UserData.readOne', function () { diff --git a/packages/compass-user-data/src/user-data.ts b/packages/compass-user-data/src/user-data.ts index 768de9e8e89..92d3cd36d5e 100644 --- a/packages/compass-user-data/src/user-data.ts +++ b/packages/compass-user-data/src/user-data.ts @@ -69,7 +69,7 @@ export class UserData { private readonly serialize: SerializeContent>; private readonly deserialize: DeserializeContent; private readonly getFileName: GetFileName; - private readonly semaphore = new Semaphore(1000); + private readonly semaphore = new Semaphore(100); constructor( private readonly validator: T,