From 20e466417ac687afbfb8b3145d7fda96860c001c Mon Sep 17 00:00:00 2001 From: gagik Date: Fri, 10 Oct 2025 18:31:49 +0200 Subject: [PATCH 01/12] chore(mongodb-downloader): use a lockfile to prevent redundant parallel downloads --- package-lock.json | 71 +++ packages/mongodb-downloader/package.json | 10 +- .../src/download-integration.spec.ts | 126 ++++++ packages/mongodb-downloader/src/index.ts | 288 ++++++++----- .../mongodb-downloader/src/npm-with-lock.ts | 406 ++++++++++++++++++ 5 files changed, 780 insertions(+), 121 deletions(-) create mode 100644 packages/mongodb-downloader/src/download-integration.spec.ts create mode 100644 packages/mongodb-downloader/src/npm-with-lock.ts diff --git a/package-lock.json b/package-lock.json index a0b7f40f..0a177909 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10367,6 +10367,16 @@ "integrity": "sha512-+68kP9yzs4LMp7VNh8gdzMSPZFL44MLGqiHWvttYJe+6qnuVr4Ek9wSBQoveqY/r+LwjCcU29kNVkidwim+kYA==", "dev": true }, + "node_modules/@types/promise-retry": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/@types/promise-retry/-/promise-retry-1.1.6.tgz", + "integrity": "sha512-EC1+OMXV0PZb0pf+cmyxc43MEP2CDumZe4AfuxWboxxEixztIebknpJPZAX5XlodGF1OY+C1E/RAeNGzxf+bJA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/retry": "*" + } + }, "node_modules/@types/prop-types": { "version": "15.7.11", "resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.11.tgz", @@ -10426,6 +10436,13 @@ "@types/node": "*" } }, + "node_modules/@types/retry": { + "version": "0.12.5", + "resolved": "https://registry.npmjs.org/@types/retry/-/retry-0.12.5.tgz", + "integrity": "sha512-3xSjTp3v03X/lSQLkczaN9UIEwJMoMCA1+Nb5HfbJEQWogdeQIyVtTvxPXDQjZ5zws8rFQfVfRdz03ARihPJgw==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/s3rver": { "version": "3.7.0", "resolved": "https://registry.npmjs.org/@types/s3rver/-/s3rver-3.7.0.tgz", @@ -10447,6 +10464,13 @@ "integrity": "sha512-k107IF4+Xr7UHjwDc7Cfd6PRQfbdkiRabXGRjo07b4WyPahFBZCZ1sE+BNxYIJPPg73UkfOsVOLwqVc/6ETrIA==", "license": "MIT" }, + "node_modules/@types/signal-exit": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@types/signal-exit/-/signal-exit-3.0.4.tgz", + "integrity": "sha512-e7EUPfU9afHyWc5CXtlqbvVHEshrb05uPlDCenWIbMgtWoFrTuTDVYNLKk6o4X2/4oHTfNqrJX/vaJ3uBhtXTg==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/sinon": { "version": "9.0.11", "resolved": "https://registry.npmjs.org/@types/sinon/-/sinon-9.0.11.tgz", @@ -25379,6 +25403,7 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/promise-retry/-/promise-retry-2.0.1.tgz", "integrity": "sha512-y+WKFlBR8BGXnsNlIHFGPZmyDf3DFMoLhaflAnyZgV6rG6xu+JwesTo2Q9R6XwYmtmwAFCkAk3e35jEdoeh/3g==", + "license": "MIT", "dependencies": { "err-code": "^2.0.2", "retry": "^0.12.0" @@ -30686,6 +30711,8 @@ "decompress": "^4.2.1", "mongodb-download-url": "^1.6.3", "node-fetch": "^2.7.0", + "promise-retry": "^2.0.1", + "signal-exit": "^4.1.0", "tar": "^6.1.15" }, "devDependencies": { @@ -30697,6 +30724,8 @@ "@types/decompress": "^4.2.4", "@types/mocha": "^9.1.1", "@types/node": "^22.15.30", + "@types/promise-retry": "^1.1.6", + "@types/signal-exit": "^3.0.4", "@types/tar": "^6.1.5", "depcheck": "^1.4.7", "eslint": "^7.25.0", @@ -30735,6 +30764,18 @@ } } }, + "packages/mongodb-downloader/node_modules/signal-exit": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", + "license": "ISC", + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "packages/mongodb-downloader/node_modules/tar": { "version": "6.2.1", "resolved": "https://registry.npmjs.org/tar/-/tar-6.2.1.tgz", @@ -38900,6 +38941,8 @@ "@types/decompress": "^4.2.4", "@types/mocha": "^9.1.1", "@types/node": "^22.15.30", + "@types/promise-retry": "^1.1.6", + "@types/signal-exit": "^3.0.4", "@types/tar": "^6.1.5", "debug": "^4.4.0", "decompress": "^4.2.1", @@ -38911,6 +38954,8 @@ "node-fetch": "^2.7.0", "nyc": "^15.1.0", "prettier": "^3.5.3", + "promise-retry": "^2.0.1", + "signal-exit": "^4.1.0", "tar": "^6.1.15", "typescript": "^5.0.4" }, @@ -38928,6 +38973,11 @@ "whatwg-url": "^5.0.0" } }, + "signal-exit": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==" + }, "tar": { "version": "6.2.1", "resolved": "https://registry.npmjs.org/tar/-/tar-6.2.1.tgz", @@ -41699,6 +41749,15 @@ "integrity": "sha512-+68kP9yzs4LMp7VNh8gdzMSPZFL44MLGqiHWvttYJe+6qnuVr4Ek9wSBQoveqY/r+LwjCcU29kNVkidwim+kYA==", "dev": true }, + "@types/promise-retry": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/@types/promise-retry/-/promise-retry-1.1.6.tgz", + "integrity": "sha512-EC1+OMXV0PZb0pf+cmyxc43MEP2CDumZe4AfuxWboxxEixztIebknpJPZAX5XlodGF1OY+C1E/RAeNGzxf+bJA==", + "dev": true, + "requires": { + "@types/retry": "*" + } + }, "@types/prop-types": { "version": "15.7.11", "resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.11.tgz", @@ -41757,6 +41816,12 @@ "@types/node": "*" } }, + "@types/retry": { + "version": "0.12.5", + "resolved": "https://registry.npmjs.org/@types/retry/-/retry-0.12.5.tgz", + "integrity": "sha512-3xSjTp3v03X/lSQLkczaN9UIEwJMoMCA1+Nb5HfbJEQWogdeQIyVtTvxPXDQjZ5zws8rFQfVfRdz03ARihPJgw==", + "dev": true + }, "@types/s3rver": { "version": "3.7.0", "resolved": "https://registry.npmjs.org/@types/s3rver/-/s3rver-3.7.0.tgz", @@ -41777,6 +41842,12 @@ "resolved": "https://registry.npmjs.org/@types/semver/-/semver-7.7.0.tgz", "integrity": "sha512-k107IF4+Xr7UHjwDc7Cfd6PRQfbdkiRabXGRjo07b4WyPahFBZCZ1sE+BNxYIJPPg73UkfOsVOLwqVc/6ETrIA==" }, + "@types/signal-exit": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@types/signal-exit/-/signal-exit-3.0.4.tgz", + "integrity": "sha512-e7EUPfU9afHyWc5CXtlqbvVHEshrb05uPlDCenWIbMgtWoFrTuTDVYNLKk6o4X2/4oHTfNqrJX/vaJ3uBhtXTg==", + "dev": true + }, "@types/sinon": { "version": "9.0.11", "resolved": "https://registry.npmjs.org/@types/sinon/-/sinon-9.0.11.tgz", diff --git a/packages/mongodb-downloader/package.json b/packages/mongodb-downloader/package.json index 2b1da795..2fbfb9da 100644 --- a/packages/mongodb-downloader/package.json +++ b/packages/mongodb-downloader/package.json @@ -53,10 +53,12 @@ }, "dependencies": { "debug": "^4.4.0", - "tar": "^6.1.15", "decompress": "^4.2.1", - "mongodb-download-url": "^1.6.3", - "node-fetch": "^2.7.0" + "node-fetch": "^2.7.0", + "promise-retry": "^2.0.1", + "signal-exit": "^4.1.0", + "tar": "^6.1.15", + "mongodb-download-url": "^1.6.3" }, "devDependencies": { "@mongodb-js/eslint-config-devtools": "0.9.12", @@ -67,6 +69,8 @@ "@types/decompress": "^4.2.4", "@types/mocha": "^9.1.1", "@types/node": "^22.15.30", + "@types/promise-retry": "^1.1.6", + "@types/signal-exit": "^3.0.4", "@types/tar": "^6.1.5", "depcheck": "^1.4.7", "eslint": "^7.25.0", diff --git a/packages/mongodb-downloader/src/download-integration.spec.ts b/packages/mongodb-downloader/src/download-integration.spec.ts new file mode 100644 index 00000000..eebd08e1 --- /dev/null +++ b/packages/mongodb-downloader/src/download-integration.spec.ts @@ -0,0 +1,126 @@ +import { expect } from 'chai'; +import { promises as fs } from 'fs'; +import path from 'path'; +import os from 'os'; +import { MongoDbDownloader } from './index'; + +describe('downloader with Locking', function () { + this.timeout(60000); + + let tmpDir: string; + + beforeEach(async function () { + tmpDir = path.join(os.tmpdir(), `download-integration-tests-${Date.now()}`); + await fs.mkdir(tmpDir, { recursive: true }); + }); + const version = '8.2.0'; + + afterEach(async function () { + try { + await fs.rm(tmpDir, { recursive: true }); + } catch { + // Ignore cleanup errors + } + }); + + it('should prevent concurrent downloads of the same version', async function () { + const downloader = new MongoDbDownloader({ tmpdir: tmpDir }); + + const results = await Promise.all([ + downloader.downloadMongoDbWithVersionInfo(version), + downloader.downloadMongoDbWithVersionInfo(version), + downloader.downloadMongoDbWithVersionInfo(version), + ]); + + // All results should be identical + expect(results[0].version).to.equal(version); + expect(results[1].version).to.equal(version); + expect(results[2].version).to.equal(version); + + expect(results[0].downloadedBinDir).to.equal(results[1].downloadedBinDir); + expect(results[1].downloadedBinDir).to.equal(results[2].downloadedBinDir); + + // Verify the downloaded directory exists and contains mongod + expect(await fs.stat(results[0].downloadedBinDir)).to.be.ok; + expect(await fs.stat(path.join(results[0].downloadedBinDir, 'mongod'))).to + .be.ok; + }); + + it('should wait for existing download to complete', async function () { + // First, download MongoDB normally + const downloader = new MongoDbDownloader({ tmpdir: tmpDir }); + const result = await downloader.downloadMongoDbWithVersionInfo(version); + + expect(result.version).to.equal(version); + expect(result.downloadedBinDir).to.be.a('string'); + + // Verify the downloaded directory exists and contains mongod + expect(await fs.stat(result.downloadedBinDir)).to.be.ok; + expect(await fs.stat(path.join(result.downloadedBinDir, 'mongod'))).to.be + .ok; + }); + + it('should skip download if already completed', async function () { + // First download + const downloader = new MongoDbDownloader({ tmpdir: tmpDir }); + const result1 = await downloader.downloadMongoDbWithVersionInfo(version); + + // Second download should use cached result + const result2 = await downloader.downloadMongoDbWithVersionInfo(version); + + expect(result1.version).to.equal(version); + expect(result2.version).to.equal(version); + expect(result1.downloadedBinDir).to.equal(result2.downloadedBinDir); + + // Verify the downloaded directory exists and contains mongod + expect(await fs.stat(result1.downloadedBinDir)).to.be.ok; + expect(await fs.stat(path.join(result1.downloadedBinDir, 'mongod'))).to.be + .ok; + }); + + it('should handle different versions independently', async function () { + const version2 = '8.1.0'; + + // Download different versions + const downloader = new MongoDbDownloader({ tmpdir: tmpDir }); + const [result1, result2] = await Promise.all([ + downloader.downloadMongoDbWithVersionInfo(version), + downloader.downloadMongoDbWithVersionInfo(version2), + ]); + + expect(result1.version).to.equal(version); + expect(result2.version).to.equal(version2); + expect(result1.downloadedBinDir).to.not.equal(result2.downloadedBinDir); + + // Verify both downloaded directories exist and contain mongod + expect(await fs.stat(result1.downloadedBinDir)).to.be.ok; + expect(await fs.stat(path.join(result1.downloadedBinDir, 'mongod'))).to.be + .ok; + expect(await fs.stat(result2.downloadedBinDir)).to.be.ok; + expect(await fs.stat(path.join(result2.downloadedBinDir, 'mongod'))).to.be + .ok; + }); + + it('should handle promise caching correctly', async function () { + const version = '8.2.0'; + + // Start multiple downloads in sequence (not parallel) + const downloader = new MongoDbDownloader({ tmpdir: tmpDir }); + const result1 = await downloader.downloadMongoDbWithVersionInfo(version); + const result2 = await downloader.downloadMongoDbWithVersionInfo(version); + const result3 = await downloader.downloadMongoDbWithVersionInfo(version); + + // All should return the same result + expect(result1.version).to.equal(version); + expect(result2.version).to.equal(version); + expect(result3.version).to.equal(version); + + expect(result1.downloadedBinDir).to.equal(result2.downloadedBinDir); + expect(result2.downloadedBinDir).to.equal(result3.downloadedBinDir); + + // Verify the downloaded directory exists and contains mongod + expect(await fs.stat(result1.downloadedBinDir)).to.be.ok; + expect(await fs.stat(path.join(result1.downloadedBinDir, 'mongod'))).to.be + .ok; + }); +}); diff --git a/packages/mongodb-downloader/src/index.ts b/packages/mongodb-downloader/src/index.ts index b554ed14..f48f3e81 100644 --- a/packages/mongodb-downloader/src/index.ts +++ b/packages/mongodb-downloader/src/index.ts @@ -12,7 +12,8 @@ import type { DownloadArtifactInfo, } from 'mongodb-download-url'; import createDebug from 'debug'; -const debug = createDebug('mongodb-downloader'); +import { withLock } from './npm-with-lock'; +export const debug = createDebug('mongodb-downloader'); export type { DownloadOptions }; @@ -20,144 +21,195 @@ export type DownloadResult = DownloadArtifactInfo & { downloadedBinDir: string; }; -// Download mongod + mongos and return the path to a directory containing them. -export async function downloadMongoDbWithVersionInfo( - tmpdir: string, - targetVersionSemverSpecifier = '*', - options: DownloadOptions = {}, -): Promise { - let wantsEnterprise = options.enterprise ?? false; - const isWindows = ['win32', 'windows'].includes( - options.platform ?? process.platform, - ); - async function lookupDownloadUrl(): Promise { +export class MongoDbDownloader { + private tmpdir: string; + + constructor({ tmpdir }: { tmpdir: string }) { + this.tmpdir = tmpdir; + } + + private downloadPromises: Record> = + Object.create(null); + + // Download mongod + mongos and return the path to a directory containing them. + async downloadMongoDbWithVersionInfo( + targetVersion = '*', + options: DownloadOptions = {}, + ): Promise { + await fs.mkdir(this.tmpdir, { recursive: true }); + if (targetVersion === 'latest-alpha') { + return await this.doDownload('latest-alpha', options); + } + + return await this.doDownload(targetVersion, options); + } + + private async lookupDownloadUrl({ + targetVersion, + enterprise, + options, + }: { + targetVersion: string; + enterprise: boolean; + options: DownloadOptions; + }): Promise { return await getDownloadURL({ - version: targetVersionSemverSpecifier, - enterprise: wantsEnterprise, + version: targetVersion, + enterprise, ...options, }); } - await fs.mkdir(tmpdir, { recursive: true }); - if (targetVersionSemverSpecifier === 'latest-alpha') { - return await doDownload( - tmpdir, - !!options.crypt_shared, - 'latest-alpha', - isWindows, - lookupDownloadUrl, + private async doDownload( + version: string, + options: DownloadOptions, + ): Promise { + const isWindows = ['win32', 'windows'].includes( + options.platform ?? process.platform, ); - } + const isCryptLibrary = !!options.crypt_shared; + const isEnterprise = options.enterprise ?? false; - if (/-enterprise$/.test(targetVersionSemverSpecifier)) { - wantsEnterprise = true; - targetVersionSemverSpecifier = targetVersionSemverSpecifier.replace( - /-enterprise$/, - '', + const downloadTarget = path.resolve( + this.tmpdir, + `mongodb-${process.platform}-${process.env.DISTRO_ID || 'none'}-${ + process.arch + }-${version}`.replace(/[^a-zA-Z0-9_-]/g, ''), ); - } + return (this.downloadPromises[downloadTarget] ??= (async () => { + const bindir = path.resolve( + downloadTarget, + isCryptLibrary && !isWindows ? 'lib' : 'bin', + ); - return await doDownload( - tmpdir, - !!options.crypt_shared, - targetVersionSemverSpecifier + - (wantsEnterprise ? '-enterprise' : '-community'), - isWindows, - () => lookupDownloadUrl(), - ); -} + const artifactInfoFile = path.join(bindir, '.artifact_info'); + const lockPath = `${downloadTarget}.lock`; -const downloadPromises: Record> = Object.create( - null, -); -async function doDownload( - tmpdir: string, - isCryptLibrary: boolean, - version: string, - isWindows: boolean, - lookupDownloadUrl: () => Promise, -): Promise { - const downloadTarget = path.resolve( - tmpdir, - `mongodb-${process.platform}-${process.env.DISTRO_ID || 'none'}-${ - process.arch - }-${version}`.replace(/[^a-zA-Z0-9_-]/g, ''), - ); - return (downloadPromises[downloadTarget] ??= (async () => { - const bindir = path.resolve( - downloadTarget, - isCryptLibrary && !isWindows ? 'lib' : 'bin', - ); - const artifactInfoFile = path.join(bindir, '.artifact_info'); + // Check if already downloaded before acquiring lock + const currentDownloadedFile = await this.getCurrentDownloadedFile({ + bindir, + artifactInfoFile, + }); + if (currentDownloadedFile) { + debug(`Skipping download because ${downloadTarget} exists`); + return currentDownloadedFile; + } + + // Acquire the lock and perform download + return await withLock(lockPath, async (signal) => { + // Check again inside lock in case another process downloaded it + const downloadedFile = await this.getCurrentDownloadedFile({ + bindir, + artifactInfoFile, + }); + if (downloadedFile) { + debug( + `Skipping download because ${downloadTarget} exists (checked inside lock)`, + ); + return downloadedFile; + } + + await fs.mkdir(downloadTarget, { recursive: true }); + const artifactInfo = await this.lookupDownloadUrl({ + targetVersion: version, + enterprise: isEnterprise, + options, + }); + const { url } = artifactInfo; + debug(`Downloading ${url} into ${downloadTarget}...`); + + await this.downloadAndExtract({ + url, + downloadTarget, + isCryptLibrary, + bindir, + }); + await fs.writeFile(artifactInfoFile, JSON.stringify(artifactInfo)); + debug(`Download complete`, bindir); + return { ...artifactInfo, downloadedBinDir: bindir }; + }); + })()); + } + + private async getCurrentDownloadedFile({ + bindir, + artifactInfoFile, + }: { + bindir: string; + artifactInfoFile: string; + }): Promise { try { await fs.stat(artifactInfoFile); - debug(`Skipping download because ${downloadTarget} exists`); return { ...JSON.parse(await fs.readFile(artifactInfoFile, 'utf8')), downloadedBinDir: bindir, }; } catch { - /* ignore */ + /* ignore - file doesn't exist, proceed with download */ + } + } + + // Using a large highWaterMark setting noticeably speeds up Windows downloads + private static HWM = 1024 * 1024; + + // eslint-disable-next-line no-inner-declarations + private async downloadAndExtract({ + withExtraStripDepth = 0, + downloadTarget, + isCryptLibrary, + bindir, + url, + }: { + withExtraStripDepth?: number; + downloadTarget: string; + isCryptLibrary: boolean; + bindir: string; + url: string; + }): Promise { + const response = await fetch(url, { + highWaterMark: MongoDbDownloader.HWM, + } as Parameters[1]); + if (/\.tgz$|\.tar(\.[^.]+)?$/.exec(url)) { + // the server's tarballs can contain hard links, which the (unmaintained?) + // `download` package is unable to handle (https://github.com/kevva/decompress/issues/93) + await promisify(pipeline)( + response.body, + tar.x({ cwd: downloadTarget, strip: isCryptLibrary ? 0 : 1 }), + ); + } else { + const filename = path.join( + downloadTarget, + path.basename(new URL(url).pathname), + ); + await promisify(pipeline)( + response.body, + createWriteStream(filename, { highWaterMark: MongoDbDownloader.HWM }), + ); + debug(`Written file ${url} to ${filename}, extracting...`); + await decompress(filename, downloadTarget, { + strip: isCryptLibrary ? 0 : 1, + filter: (file) => path.extname(file.path) !== '.pdb', // Windows .pdb files are huge and useless + }); } - await fs.mkdir(downloadTarget, { recursive: true }); - const artifactInfo = await lookupDownloadUrl(); - const { url } = artifactInfo; - debug(`Downloading ${url} into ${downloadTarget}...`); - - // Using a large highWaterMark setting noticeably speeds up Windows downloads - const HWM = 1024 * 1024; - async function downloadAndExtract(withExtraStripDepth = 0): Promise { - const response = await fetch(url, { - highWaterMark: HWM, - } as any); - if (/\.tgz$|\.tar(\.[^.]+)?$/.exec(url)) { - // the server's tarballs can contain hard links, which the (unmaintained?) - // `download` package is unable to handle (https://github.com/kevva/decompress/issues/93) - await promisify(pipeline)( - response.body, - tar.x({ cwd: downloadTarget, strip: isCryptLibrary ? 0 : 1 }), - ); - } else { - const filename = path.join( + try { + await fs.stat(bindir); // Make sure it exists. + } catch (err) { + if (withExtraStripDepth === 0 && url.includes('macos')) { + // The server team changed how macos release artifacts are packed + // and added a `./` prefix to paths in the tarball, + // which seems like it shouldn't change anything but does + // in fact require an increased path strip depth. + console.info('Retry due to miscalculated --strip-components depth'); + return await this.downloadAndExtract({ + withExtraStripDepth: 1, + url, downloadTarget, - path.basename(new URL(url).pathname), - ); - await promisify(pipeline)( - response.body, - createWriteStream(filename, { highWaterMark: HWM }), - ); - debug(`Written file ${url} to ${filename}, extracting...`); - await decompress(filename, downloadTarget, { - strip: isCryptLibrary ? 0 : 1, - filter: (file) => path.extname(file.path) !== '.pdb', // Windows .pdb files are huge and useless + isCryptLibrary, + bindir, }); } - - try { - await fs.stat(bindir); // Make sure it exists. - } catch (err) { - if (withExtraStripDepth === 0 && url.includes('macos')) { - // The server team changed how macos release artifacts are packed - // and added a `./` prefix to paths in the tarball, - // which seems like it shouldn't change anything but does - // in fact require an increased path strip depth. - console.info('Retry due to miscalculated --strip-components depth'); - return await downloadAndExtract(1); - } - throw err; - } + throw err; } - - await downloadAndExtract(); - await fs.writeFile(artifactInfoFile, JSON.stringify(artifactInfo)); - debug(`Download complete`, bindir); - return { ...artifactInfo, downloadedBinDir: bindir }; - })()); -} - -export async function downloadMongoDb( - ...args: Parameters -): Promise { - return (await downloadMongoDbWithVersionInfo(...args)).downloadedBinDir; + } } diff --git a/packages/mongodb-downloader/src/npm-with-lock.ts b/packages/mongodb-downloader/src/npm-with-lock.ts new file mode 100644 index 00000000..7f2afebb --- /dev/null +++ b/packages/mongodb-downloader/src/npm-with-lock.ts @@ -0,0 +1,406 @@ +// Adapted from: +// https://raw.githubusercontent.com/npm/cli/072253549d774893a3689341dbc660cb845ebcfe/workspaces/libnpmexec/lib/with-lock.js + +// The Artistic License 2.0 +// Copyright (c) 2000-2006, The Perl Foundation. +// +// Everyone is permitted to copy and distribute verbatim copies +// of this license document, but changing it is not allowed. +// +// Preamble +// +// This license establishes the terms under which a given free software +// Package may be copied, modified, distributed, and/or redistributed. +// The intent is that the Copyright Holder maintains some artistic +// control over the development of that Package while still keeping the +// Package available as open source and free software. + +// You are always permitted to make arrangements wholly outside of this +// license directly with the Copyright Holder of a given Package. If the +// terms of this license do not permit the full use that you propose to +// make of the Package, you should contact the Copyright Holder and seek +// a different licensing arrangement. +// +// Definitions +// +// "Copyright Holder" means the individual(s) or organization(s) +// named in the copyright notice for the entire Package. +// +// "Contributor" means any party that has contributed code or other +// material to the Package, in accordance with the Copyright Holder's +// procedures. +// +// "You" and "your" means any person who would like to copy, +// distribute, or modify the Package. +// +// "Package" means the collection of files distributed by the +// Copyright Holder, and derivatives of that collection and/or of +// those files. A given Package may consist of either the Standard +// Version, or a Modified Version. +// +// "Distribute" means providing a copy of the Package or making it +// accessible to anyone else, or in the case of a company or +// organization, to others outside of your company or organization. +// +// "Distributor Fee" means any fee that you charge for Distributing +// this Package or providing support for this Package to another +// party. It does not mean licensing fees. +// +// "Standard Version" refers to the Package if it has not been +// modified, or has been modified only in ways explicitly requested +// by the Copyright Holder. +// +// "Modified Version" means the Package, if it has been changed, and +// such changes were not explicitly requested by the Copyright +// Holder. +// +// "Original License" means this Artistic License as Distributed with +// the Standard Version of the Package, in its current version or as +// it may be modified by The Perl Foundation in the future. +// +// "Source" form means the source code, documentation source, and +// configuration files for the Package. +// +// "Compiled" form means the compiled bytecode, object code, binary, +// or any other form resulting from mechanical transformation or +// translation of the Source form. +// +// Permission for Use and Modification Without Distribution + +// (1) You are permitted to use the Standard Version and create and use +// Modified Versions for any purpose without restriction, provided that +// you do not Distribute the Modified Version. +// +// Permissions for Redistribution of the Standard Version + +// (2) You may Distribute verbatim copies of the Source form of the +// Standard Version of this Package in any medium without restriction, +// either gratis or for a Distributor Fee, provided that you duplicate +// all of the original copyright notices and associated disclaimers. At +// your discretion, such verbatim copies may or may not include a +// Compiled form of the Package. + +// (3) You may apply any bug fixes, portability changes, and other +// modifications made available from the Copyright Holder. The resulting +// Package will still be considered the Standard Version, and as such +// will be subject to the Original License. +// +// Distribution of Modified Versions of the Package as Source +// +// (4) You may Distribute your Modified Version as Source (either gratis +// or for a Distributor Fee, and with or without a Compiled form of the +// Modified Version) provided that you clearly document how it differs +// from the Standard Version, including, but not limited to, documenting +// any non-standard features, executables, or modules, and provided that +// you do at least ONE of the following: +// +// (a) make the Modified Version available to the Copyright Holder +// of the Standard Version, under the Original License, so that the +// Copyright Holder may include your modifications in the Standard +// Version. +// +// (b) ensure that installation of your Modified Version does not +// prevent the user installing or running the Standard Version. In +// addition, the Modified Version must bear a name that is different +// from the name of the Standard Version. +// +// (c) allow anyone who receives a copy of the Modified Version to +// make the Source form of the Modified Version available to others +// under + +// (i) the Original License or +// +// (ii) a license that permits the licensee to freely copy, +// modify and redistribute the Modified Version using the same +// licensing terms that apply to the copy that the licensee +// received, and requires that the Source form of the Modified +// Version, and of any works derived from it, be made freely +// available in that license fees are prohibited but Distributor +// Fees are allowed. +// +// Distribution of Compiled Forms of the Standard Version +// or Modified Versions without the Source +// +// (5) You may Distribute Compiled forms of the Standard Version without +// the Source, provided that you include complete instructions on how to +// get the Source of the Standard Version. Such instructions must be +// valid at the time of your distribution. If these instructions, at any +// time while you are carrying out such distribution, become invalid, you +// must provide new instructions on demand or cease further distribution. +// If you provide valid instructions or cease distribution within thirty +// days after you become aware that the instructions are invalid, then +// you do not forfeit any of your rights under this license. +// +// (6) You may Distribute a Modified Version in Compiled form without +// the Source, provided that you comply with Section 4 with respect to +// the Source of the Modified Version. +// +// Aggregating or Linking the Package + +// (7) You may aggregate the Package (either the Standard Version or +// Modified Version) with other packages and Distribute the resulting +// aggregation provided that you do not charge a licensing fee for the +// Package. Distributor Fees are permitted, and licensing fees for other +// components in the aggregation are permitted. The terms of this license +// apply to the use and Distribution of the Standard or Modified Versions +// as included in the aggregation. +// +// (8) You are permitted to link Modified and Standard Versions with +// other works, to embed the Package in a larger work of your own, or to +// build stand-alone binary or bytecode versions of applications that +// include the Package, and Distribute the result without restriction, +// provided the result does not expose a direct interface to the Package. +// +// Items That are Not Considered Part of a Modified Version + +// (9) Works (including, but not limited to, modules and scripts) that +// merely extend or make use of the Package, do not, by themselves, cause +// the Package to be a Modified Version. In addition, such works are not +// considered parts of the Package itself, and are not subject to the +// terms of this license. +// +// General Provisions + +// (10) Any use, modification, and distribution of the Standard or +// Modified Versions is governed by this Artistic License. By using, +// modifying or distributing the Package, you accept this license. Do not +// use, modify, or distribute the Package, if you do not accept this +// license. +// +// (11) If your Modified Version has been derived from a Modified +// Version made by someone other than you, you are nevertheless required +// to ensure that your Modified Version complies with the requirements of +// this license. +// +// (12) This license does not grant you the right to use any trademark, +// service mark, tradename, or logo of the Copyright Holder. +// +// (13) This license includes the non-exclusive, worldwide, +// free-of-charge patent license to make, have made, use, offer to sell, +// sell, import and otherwise transfer the Package with respect to any +// patent claims licensable by the Copyright Holder that are necessarily +// infringed by the Package. If you institute patent litigation +// (including a cross-claim or counterclaim) against any party alleging +// that the Package constitutes direct or contributory patent +// infringement, then this Artistic License to you shall terminate on the +// date that such litigation is filed. +// +// (14) Disclaimer of Warranty: +// THE PACKAGE IS PROVIDED BY THE COPYRIGHT HOLDER AND CONTRIBUTORS "AS +// IS' AND WITHOUT ANY EXPRESS OR IMPLIED WARRANTIES. THE IMPLIED +// WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, OR +// NON-INFRINGEMENT ARE DISCLAIMED TO THE EXTENT PERMITTED BY YOUR LOCAL +// LAW. UNLESS REQUIRED BY LAW, NO COPYRIGHT HOLDER OR CONTRIBUTOR WILL +// BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL +// DAMAGES ARISING IN ANY WAY OUT OF THE USE OF THE PACKAGE, EVEN IF +// ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import fs from 'node:fs/promises'; +import { rmdirSync } from 'node:fs'; +import promiseRetry from 'promise-retry'; +import { onExit } from 'signal-exit'; + +// a lockfile implementation inspired by the unmaintained proper-lockfile library +// +// similarities: +// - based on mkdir's atomicity +// - works across processes and even machines (via NFS) +// - cleans up after itself +// - detects compromised locks +// +// differences: +// - higher-level API (just a withLock function) +// - written in async/await style +// - uses mtime + inode for more reliable compromised lock detection +// - more ergonomic compromised lock handling (i.e. withLock will reject, and callbacks have access to an AbortSignal) +// - uses a more recent version of signal-exit + +const touchInterval = 1_000; +// mtime precision is platform dependent, so use a reasonably large threshold +const staleThreshold = 5_000; + +// track current locks and their cleanup functions +const currentLocks = new Map void>(); + +function cleanupLocks() { + for (const [, cleanup] of currentLocks) { + try { + cleanup(); + } catch (err) { + // + } + } +} + +// clean up any locks that were not released normally +onExit(cleanupLocks); + +/** + * Acquire an advisory lock for the given path and hold it for the duration of the callback. + * + * The lock will be released automatically when the callback resolves or rejects. + * Concurrent calls to withLock() for the same path will wait until the lock is released. + */ +export async function withLock( + lockPath: string, + cb: (signal: AbortSignal) => Promise, +): Promise { + try { + const signal = await acquireLock(lockPath); + return await new Promise((resolve, reject) => { + signal.addEventListener('abort', () => { + reject( + Object.assign(new Error('Lock compromised'), { + code: 'ECOMPROMISED', + }), + ); + }); + + void (async () => { + try { + resolve(await cb(signal)); + } catch (err) { + reject(err); + } + })(); + }); + } finally { + releaseLock(lockPath); + } +} + +function acquireLock(lockPath: string): Promise { + return promiseRetry( + { + minTimeout: 100, + maxTimeout: 5_000, + // if another process legitimately holds the lock, wait for it to release; if it dies abnormally and the lock becomes stale, we'll acquire it automatically + forever: true, + }, + async (retry: (err: unknown) => never) => { + try { + await fs.mkdir(lockPath); + } catch (err: unknown) { + if ( + err && + typeof err === 'object' && + 'code' in err && + err.code !== 'EEXIST' && + err.code !== 'EBUSY' && + err.code !== 'EPERM' + ) { + throw err; + } + + const status = await getLockStatus(lockPath); + + if (status === 'locked') { + // let's see if we can acquire it on the next attempt 🤞 + return retry(err); + } + if (status === 'stale') { + try { + // there is a very tiny window where another process could also release the stale lock and acquire it before we release it here; the lock compromise checker should detect this and throw an error + deleteLock(lockPath); + } catch (e: unknown) { + // on windows, EBUSY/EPERM can happen if another process is (re)creating the lock; maybe we can acquire it on a subsequent attempt 🤞 + if ( + e && + typeof e === 'object' && + 'code' in e && + (e.code === 'EBUSY' || e.code === 'EPERM') + ) { + return retry(e); + } + throw e; + } + } + // immediately attempt to acquire the lock (no backoff) + return await acquireLock(lockPath); + } + try { + const signal = await maintainLock(lockPath); + return signal; + } catch (err) { + throw Object.assign(new Error('Lock compromised'), { + code: 'ECOMPROMISED', + }); + } + }, + ); +} + +function deleteLock(lockPath: string): void { + try { + // synchronous, so we can call in an exit handler + rmdirSync(lockPath); + } catch (err: unknown) { + if ( + err && + typeof err === 'object' && + 'code' in err && + err.code !== 'ENOENT' + ) { + throw err; + } + } +} + +function releaseLock(lockPath: string): void { + currentLocks.get(lockPath)?.(); + currentLocks.delete(lockPath); +} + +async function getLockStatus( + lockPath: string, +): Promise<'locked' | 'stale' | 'unlocked'> { + try { + const stat = await fs.stat(lockPath); + return Date.now() - stat.mtimeMs > staleThreshold ? 'stale' : 'locked'; + } catch (err: unknown) { + if ( + err && + typeof err === 'object' && + 'code' in err && + err.code === 'ENOENT' + ) { + return 'unlocked'; + } + throw err; + } +} + +async function maintainLock(lockPath: string): Promise { + const controller = new AbortController(); + const stats = await fs.stat(lockPath); + // fs.utimes operates on floating points seconds (directly, or via strings/Date objects), which may not match the underlying filesystem's mtime precision, meaning that we might read a slightly different mtime than we write. always round to the nearest second, since all filesystems support at least second precision + let mtime = Math.round(stats.mtimeMs / 1000); + const signal = controller.signal; + + async function touchLock() { + try { + const currentStats = await fs.stat(lockPath); + const currentMtime = Math.round(currentStats.mtimeMs / 1000); + if (currentStats.ino !== stats.ino || currentMtime !== mtime) { + throw new Error('Lock compromised'); + } + mtime = Math.round(Date.now() / 1000); + // touch the lock, unless we just released it during this iteration + if (currentLocks.has(lockPath)) { + await fs.utimes(lockPath, mtime, mtime); + } + } catch (err: unknown) { + // stats mismatch or other fs error means the lock was compromised + controller.abort(); + } + } + + const timeout = setInterval(() => void touchLock(), touchInterval); + timeout.unref(); + function cleanup() { + clearInterval(timeout); + deleteLock(lockPath); + } + currentLocks.set(lockPath, cleanup); + return signal; +} From 1ef4dc56ed5107317cd99ed22b669071c37b3fde Mon Sep 17 00:00:00 2001 From: gagik Date: Tue, 21 Oct 2025 10:23:55 +0200 Subject: [PATCH 02/12] chore: cleanup, use proper-lockfile, use original interface --- package-lock.json | 44 ++ packages/mongodb-downloader/package.json | 6 +- ...ation.spec.ts => download-locking.spec.ts} | 77 ++-- packages/mongodb-downloader/src/index.spec.ts | 0 packages/mongodb-downloader/src/index.ts | 317 +++++++------- .../mongodb-downloader/src/npm-with-lock.ts | 406 ------------------ packages/mongodb-downloader/src/with-lock.ts | 61 +++ 7 files changed, 310 insertions(+), 601 deletions(-) rename packages/mongodb-downloader/src/{download-integration.spec.ts => download-locking.spec.ts} (56%) delete mode 100644 packages/mongodb-downloader/src/index.spec.ts delete mode 100644 packages/mongodb-downloader/src/npm-with-lock.ts create mode 100644 packages/mongodb-downloader/src/with-lock.ts diff --git a/package-lock.json b/package-lock.json index 0a177909..34b91b03 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10383,6 +10383,16 @@ "integrity": "sha512-ga8y9v9uyeiLdpKddhxYQkxNDrfvuPrlFb0N1qnZZByvcElJaXthF1UhvCh9TLWJBEHeNtdnbysW7Y6Uq8CVng==", "dev": true }, + "node_modules/@types/proper-lockfile": { + "version": "4.1.4", + "resolved": "https://registry.npmjs.org/@types/proper-lockfile/-/proper-lockfile-4.1.4.tgz", + "integrity": "sha512-uo2ABllncSqg9F1D4nugVl9v93RmjxF6LJzQLMLDdPaXCUIDPeOJ21Gbqi43xNKzBi/WQ0Q0dICqufzQbMjipQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/retry": "*" + } + }, "node_modules/@types/react": { "version": "17.0.75", "resolved": "https://registry.npmjs.org/@types/react/-/react-17.0.75.tgz", @@ -25455,6 +25465,17 @@ "node": ">= 8" } }, + "node_modules/proper-lockfile": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/proper-lockfile/-/proper-lockfile-4.1.2.tgz", + "integrity": "sha512-TjNPblN4BwAWMXU8s9AEz4JmQxnD1NNL7bNOY/AKUzyamc379FWASUhc/K1pL2noVb+XmZKLL68cjzLsiOAMaA==", + "license": "MIT", + "dependencies": { + "graceful-fs": "^4.2.4", + "retry": "^0.12.0", + "signal-exit": "^3.0.2" + } + }, "node_modules/protocols": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/protocols/-/protocols-2.0.1.tgz", @@ -30712,6 +30733,7 @@ "mongodb-download-url": "^1.6.3", "node-fetch": "^2.7.0", "promise-retry": "^2.0.1", + "proper-lockfile": "^4.1.2", "signal-exit": "^4.1.0", "tar": "^6.1.15" }, @@ -30725,6 +30747,7 @@ "@types/mocha": "^9.1.1", "@types/node": "^22.15.30", "@types/promise-retry": "^1.1.6", + "@types/proper-lockfile": "^4.1.4", "@types/signal-exit": "^3.0.4", "@types/tar": "^6.1.5", "depcheck": "^1.4.7", @@ -38942,6 +38965,7 @@ "@types/mocha": "^9.1.1", "@types/node": "^22.15.30", "@types/promise-retry": "^1.1.6", + "@types/proper-lockfile": "^4.1.4", "@types/signal-exit": "^3.0.4", "@types/tar": "^6.1.5", "debug": "^4.4.0", @@ -38955,6 +38979,7 @@ "nyc": "^15.1.0", "prettier": "^3.5.3", "promise-retry": "^2.0.1", + "proper-lockfile": "^4.1.2", "signal-exit": "^4.1.0", "tar": "^6.1.15", "typescript": "^5.0.4" @@ -41764,6 +41789,15 @@ "integrity": "sha512-ga8y9v9uyeiLdpKddhxYQkxNDrfvuPrlFb0N1qnZZByvcElJaXthF1UhvCh9TLWJBEHeNtdnbysW7Y6Uq8CVng==", "dev": true }, + "@types/proper-lockfile": { + "version": "4.1.4", + "resolved": "https://registry.npmjs.org/@types/proper-lockfile/-/proper-lockfile-4.1.4.tgz", + "integrity": "sha512-uo2ABllncSqg9F1D4nugVl9v93RmjxF6LJzQLMLDdPaXCUIDPeOJ21Gbqi43xNKzBi/WQ0Q0dICqufzQbMjipQ==", + "dev": true, + "requires": { + "@types/retry": "*" + } + }, "@types/react": { "version": "17.0.75", "resolved": "https://registry.npmjs.org/@types/react/-/react-17.0.75.tgz", @@ -53242,6 +53276,16 @@ "integrity": "sha512-vGrhOavPSTz4QVNuBNdcNXePNdNMaO1xj9yBeH1ScQPjk/rhg9sSlCXPhMkFuaNNW/syTvYqsnbIJxMBfRbbag==", "dev": true }, + "proper-lockfile": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/proper-lockfile/-/proper-lockfile-4.1.2.tgz", + "integrity": "sha512-TjNPblN4BwAWMXU8s9AEz4JmQxnD1NNL7bNOY/AKUzyamc379FWASUhc/K1pL2noVb+XmZKLL68cjzLsiOAMaA==", + "requires": { + "graceful-fs": "^4.2.4", + "retry": "^0.12.0", + "signal-exit": "^3.0.2" + } + }, "protocols": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/protocols/-/protocols-2.0.1.tgz", diff --git a/packages/mongodb-downloader/package.json b/packages/mongodb-downloader/package.json index 2fbfb9da..f94f5316 100644 --- a/packages/mongodb-downloader/package.json +++ b/packages/mongodb-downloader/package.json @@ -54,11 +54,12 @@ "dependencies": { "debug": "^4.4.0", "decompress": "^4.2.1", + "mongodb-download-url": "^1.6.3", "node-fetch": "^2.7.0", "promise-retry": "^2.0.1", + "proper-lockfile": "^4.1.2", "signal-exit": "^4.1.0", - "tar": "^6.1.15", - "mongodb-download-url": "^1.6.3" + "tar": "^6.1.15" }, "devDependencies": { "@mongodb-js/eslint-config-devtools": "0.9.12", @@ -70,6 +71,7 @@ "@types/mocha": "^9.1.1", "@types/node": "^22.15.30", "@types/promise-retry": "^1.1.6", + "@types/proper-lockfile": "^4.1.4", "@types/signal-exit": "^3.0.4", "@types/tar": "^6.1.5", "depcheck": "^1.4.7", diff --git a/packages/mongodb-downloader/src/download-integration.spec.ts b/packages/mongodb-downloader/src/download-locking.spec.ts similarity index 56% rename from packages/mongodb-downloader/src/download-integration.spec.ts rename to packages/mongodb-downloader/src/download-locking.spec.ts index eebd08e1..c58c1173 100644 --- a/packages/mongodb-downloader/src/download-integration.spec.ts +++ b/packages/mongodb-downloader/src/download-locking.spec.ts @@ -2,34 +2,35 @@ import { expect } from 'chai'; import { promises as fs } from 'fs'; import path from 'path'; import os from 'os'; -import { MongoDbDownloader } from './index'; +import { downloadMongoDbWithVersionInfo } from '.'; -describe('downloader with Locking', function () { - this.timeout(60000); +describe('downloader with locking', function () { + this.timeout(60_000); - let tmpDir: string; + let directory: string; beforeEach(async function () { - tmpDir = path.join(os.tmpdir(), `download-integration-tests-${Date.now()}`); - await fs.mkdir(tmpDir, { recursive: true }); + directory = path.join( + os.tmpdir(), + `download-integration-tests-${Date.now()}`, + ); + await fs.mkdir(directory, { recursive: true }); }); const version = '8.2.0'; afterEach(async function () { try { - await fs.rm(tmpDir, { recursive: true }); + await fs.rm(directory, { recursive: true }); } catch { // Ignore cleanup errors } }); it('should prevent concurrent downloads of the same version', async function () { - const downloader = new MongoDbDownloader({ tmpdir: tmpDir }); - const results = await Promise.all([ - downloader.downloadMongoDbWithVersionInfo(version), - downloader.downloadMongoDbWithVersionInfo(version), - downloader.downloadMongoDbWithVersionInfo(version), + downloadMongoDbWithVersionInfo({ directory, version, useLockfile: true }), + downloadMongoDbWithVersionInfo({ directory, version, useLockfile: true }), + downloadMongoDbWithVersionInfo({ directory, version, useLockfile: true }), ]); // All results should be identical @@ -48,8 +49,11 @@ describe('downloader with Locking', function () { it('should wait for existing download to complete', async function () { // First, download MongoDB normally - const downloader = new MongoDbDownloader({ tmpdir: tmpDir }); - const result = await downloader.downloadMongoDbWithVersionInfo(version); + const result = await downloadMongoDbWithVersionInfo({ + directory, + version, + useLockfile: true, + }); expect(result.version).to.equal(version); expect(result.downloadedBinDir).to.be.a('string'); @@ -62,11 +66,18 @@ describe('downloader with Locking', function () { it('should skip download if already completed', async function () { // First download - const downloader = new MongoDbDownloader({ tmpdir: tmpDir }); - const result1 = await downloader.downloadMongoDbWithVersionInfo(version); + const result1 = await downloadMongoDbWithVersionInfo({ + directory, + version, + useLockfile: true, + }); // Second download should use cached result - const result2 = await downloader.downloadMongoDbWithVersionInfo(version); + const result2 = await downloadMongoDbWithVersionInfo({ + directory, + version, + useLockfile: true, + }); expect(result1.version).to.equal(version); expect(result2.version).to.equal(version); @@ -82,10 +93,13 @@ describe('downloader with Locking', function () { const version2 = '8.1.0'; // Download different versions - const downloader = new MongoDbDownloader({ tmpdir: tmpDir }); const [result1, result2] = await Promise.all([ - downloader.downloadMongoDbWithVersionInfo(version), - downloader.downloadMongoDbWithVersionInfo(version2), + downloadMongoDbWithVersionInfo({ directory, version, useLockfile: true }), + downloadMongoDbWithVersionInfo({ + directory, + version: version2, + useLockfile: true, + }), ]); expect(result1.version).to.equal(version); @@ -100,27 +114,4 @@ describe('downloader with Locking', function () { expect(await fs.stat(path.join(result2.downloadedBinDir, 'mongod'))).to.be .ok; }); - - it('should handle promise caching correctly', async function () { - const version = '8.2.0'; - - // Start multiple downloads in sequence (not parallel) - const downloader = new MongoDbDownloader({ tmpdir: tmpDir }); - const result1 = await downloader.downloadMongoDbWithVersionInfo(version); - const result2 = await downloader.downloadMongoDbWithVersionInfo(version); - const result3 = await downloader.downloadMongoDbWithVersionInfo(version); - - // All should return the same result - expect(result1.version).to.equal(version); - expect(result2.version).to.equal(version); - expect(result3.version).to.equal(version); - - expect(result1.downloadedBinDir).to.equal(result2.downloadedBinDir); - expect(result2.downloadedBinDir).to.equal(result3.downloadedBinDir); - - // Verify the downloaded directory exists and contains mongod - expect(await fs.stat(result1.downloadedBinDir)).to.be.ok; - expect(await fs.stat(path.join(result1.downloadedBinDir, 'mongod'))).to.be - .ok; - }); }); diff --git a/packages/mongodb-downloader/src/index.spec.ts b/packages/mongodb-downloader/src/index.spec.ts deleted file mode 100644 index e69de29b..00000000 diff --git a/packages/mongodb-downloader/src/index.ts b/packages/mongodb-downloader/src/index.ts index f48f3e81..20e72fb2 100644 --- a/packages/mongodb-downloader/src/index.ts +++ b/packages/mongodb-downloader/src/index.ts @@ -12,8 +12,9 @@ import type { DownloadArtifactInfo, } from 'mongodb-download-url'; import createDebug from 'debug'; -import { withLock } from './npm-with-lock'; -export const debug = createDebug('mongodb-downloader'); +import { withLock } from './with-lock'; + +const debug = createDebug('mongodb-downloader'); export type { DownloadOptions }; @@ -21,84 +22,71 @@ export type DownloadResult = DownloadArtifactInfo & { downloadedBinDir: string; }; -export class MongoDbDownloader { - private tmpdir: string; - - constructor({ tmpdir }: { tmpdir: string }) { - this.tmpdir = tmpdir; - } - - private downloadPromises: Record> = - Object.create(null); - - // Download mongod + mongos and return the path to a directory containing them. - async downloadMongoDbWithVersionInfo( - targetVersion = '*', - options: DownloadOptions = {}, - ): Promise { - await fs.mkdir(this.tmpdir, { recursive: true }); - if (targetVersion === 'latest-alpha') { - return await this.doDownload('latest-alpha', options); - } +export type MongoDBDownloaderOptions = { + /** The directory to download the artifacts to. */ + directory: string; + /** The semantic version specifier for the target version. */ + version: string; + /** Whether to use a lockfile for preventing concurrent downloads of the same version. */ + useLockfile: boolean; + /** The options to pass to the download URL lookup. */ + downloadOptions?: DownloadOptions; +}; - return await this.doDownload(targetVersion, options); +// Download mongod + mongos and return the path to a directory containing them. +export async function downloadMongoDbWithVersionInfo({ + downloadOptions = {}, + version, + directory, + useLockfile, +}: MongoDBDownloaderOptions): Promise { + await fs.mkdir(directory, { recursive: true }); + const isWindows = ['win32', 'windows'].includes( + downloadOptions.platform ?? process.platform, + ); + const isCryptLibrary = !!downloadOptions.crypt_shared; + let isEnterprise = downloadOptions.enterprise ?? false; + let versionName = version; + + if (/-enterprise$/.test(version)) { + isEnterprise = true; + versionName = versionName.replace(/-enterprise$/, ''); } - private async lookupDownloadUrl({ - targetVersion, - enterprise, - options, - }: { - targetVersion: string; - enterprise: boolean; - options: DownloadOptions; - }): Promise { - return await getDownloadURL({ - version: targetVersion, - enterprise, - ...options, - }); + if (versionName !== 'latest-alpha') { + versionName = versionName + (isEnterprise ? '-enterprise' : '-community'); } - private async doDownload( - version: string, - options: DownloadOptions, - ): Promise { - const isWindows = ['win32', 'windows'].includes( - options.platform ?? process.platform, + const downloadTarget = path.resolve( + directory, + `mongodb-${process.platform}-${process.env.DISTRO_ID || 'none'}-${ + process.arch + }-${versionName}`.replace(/[^a-zA-Z0-9_-]/g, ''), + ); + return (async () => { + const bindir = path.resolve( + downloadTarget, + isCryptLibrary && !isWindows ? 'lib' : 'bin', ); - const isCryptLibrary = !!options.crypt_shared; - const isEnterprise = options.enterprise ?? false; - - const downloadTarget = path.resolve( - this.tmpdir, - `mongodb-${process.platform}-${process.env.DISTRO_ID || 'none'}-${ - process.arch - }-${version}`.replace(/[^a-zA-Z0-9_-]/g, ''), - ); - return (this.downloadPromises[downloadTarget] ??= (async () => { - const bindir = path.resolve( - downloadTarget, - isCryptLibrary && !isWindows ? 'lib' : 'bin', - ); - const artifactInfoFile = path.join(bindir, '.artifact_info'); - const lockPath = `${downloadTarget}.lock`; + const artifactInfoFile = path.join(bindir, '.artifact_info'); - // Check if already downloaded before acquiring lock - const currentDownloadedFile = await this.getCurrentDownloadedFile({ - bindir, - artifactInfoFile, - }); - if (currentDownloadedFile) { - debug(`Skipping download because ${downloadTarget} exists`); - return currentDownloadedFile; - } + // Check if already downloaded before acquiring lock + const currentDownloadedFile = await getCurrentDownloadedFile({ + bindir, + artifactInfoFile, + }); + if (currentDownloadedFile) { + debug(`Skipping download because ${downloadTarget} exists`); + return currentDownloadedFile; + } - // Acquire the lock and perform download - return await withLock(lockPath, async (signal) => { + // Acquire the lock and perform download + return await (useLockfile ? withLock : withoutLock)( + downloadTarget, + async () => { // Check again inside lock in case another process downloaded it - const downloadedFile = await this.getCurrentDownloadedFile({ + const downloadedFile = await getCurrentDownloadedFile({ bindir, artifactInfoFile, }); @@ -110,15 +98,15 @@ export class MongoDbDownloader { } await fs.mkdir(downloadTarget, { recursive: true }); - const artifactInfo = await this.lookupDownloadUrl({ + const artifactInfo = await lookupDownloadUrl({ targetVersion: version, enterprise: isEnterprise, - options, + options: downloadOptions, }); const { url } = artifactInfo; debug(`Downloading ${url} into ${downloadTarget}...`); - await this.downloadAndExtract({ + await downloadAndExtract({ url, downloadTarget, isCryptLibrary, @@ -127,89 +115,118 @@ export class MongoDbDownloader { await fs.writeFile(artifactInfoFile, JSON.stringify(artifactInfo)); debug(`Download complete`, bindir); return { ...artifactInfo, downloadedBinDir: bindir }; - }); - })()); - } + }, + ); + })(); +} - private async getCurrentDownloadedFile({ - bindir, - artifactInfoFile, - }: { - bindir: string; - artifactInfoFile: string; - }): Promise { - try { - await fs.stat(artifactInfoFile); - return { - ...JSON.parse(await fs.readFile(artifactInfoFile, 'utf8')), - downloadedBinDir: bindir, - }; - } catch { - /* ignore - file doesn't exist, proceed with download */ - } +const HWM = 1024 * 1024; + +async function downloadAndExtract({ + withExtraStripDepth = 0, + downloadTarget, + isCryptLibrary, + bindir, + url, +}: { + withExtraStripDepth?: number; + downloadTarget: string; + isCryptLibrary: boolean; + bindir: string; + url: string; +}): Promise { + const response = await fetch(url, { + highWaterMark: HWM, + } as Parameters[1]); + if (/\.tgz$|\.tar(\.[^.]+)?$/.exec(url)) { + // the server's tarballs can contain hard links, which the (unmaintained?) + // `download` package is unable to handle (https://github.com/kevva/decompress/issues/93) + await promisify(pipeline)( + response.body, + tar.x({ cwd: downloadTarget, strip: isCryptLibrary ? 0 : 1 }), + ); + } else { + const filename = path.join( + downloadTarget, + path.basename(new URL(url).pathname), + ); + await promisify(pipeline)( + response.body, + createWriteStream(filename, { highWaterMark: HWM }), + ); + debug(`Written file ${url} to ${filename}, extracting...`); + await decompress(filename, downloadTarget, { + strip: isCryptLibrary ? 0 : 1, + filter: (file) => path.extname(file.path) !== '.pdb', // Windows .pdb files are huge and useless + }); } - // Using a large highWaterMark setting noticeably speeds up Windows downloads - private static HWM = 1024 * 1024; - - // eslint-disable-next-line no-inner-declarations - private async downloadAndExtract({ - withExtraStripDepth = 0, - downloadTarget, - isCryptLibrary, - bindir, - url, - }: { - withExtraStripDepth?: number; - downloadTarget: string; - isCryptLibrary: boolean; - bindir: string; - url: string; - }): Promise { - const response = await fetch(url, { - highWaterMark: MongoDbDownloader.HWM, - } as Parameters[1]); - if (/\.tgz$|\.tar(\.[^.]+)?$/.exec(url)) { - // the server's tarballs can contain hard links, which the (unmaintained?) - // `download` package is unable to handle (https://github.com/kevva/decompress/issues/93) - await promisify(pipeline)( - response.body, - tar.x({ cwd: downloadTarget, strip: isCryptLibrary ? 0 : 1 }), - ); - } else { - const filename = path.join( + try { + await fs.stat(bindir); // Make sure it exists. + } catch (err) { + if (withExtraStripDepth === 0 && url.includes('macos')) { + // The server team changed how macos release artifacts are packed + // and added a `./` prefix to paths in the tarball, + // which seems like it shouldn't change anything but does + // in fact require an increased path strip depth. + // eslint-disable-next-line no-console + console.info('Retry due to miscalculated --strip-components depth'); + return await downloadAndExtract({ + withExtraStripDepth: 1, + url, downloadTarget, - path.basename(new URL(url).pathname), - ); - await promisify(pipeline)( - response.body, - createWriteStream(filename, { highWaterMark: MongoDbDownloader.HWM }), - ); - debug(`Written file ${url} to ${filename}, extracting...`); - await decompress(filename, downloadTarget, { - strip: isCryptLibrary ? 0 : 1, - filter: (file) => path.extname(file.path) !== '.pdb', // Windows .pdb files are huge and useless + isCryptLibrary, + bindir, }); } + throw err; + } +} - try { - await fs.stat(bindir); // Make sure it exists. - } catch (err) { - if (withExtraStripDepth === 0 && url.includes('macos')) { - // The server team changed how macos release artifacts are packed - // and added a `./` prefix to paths in the tarball, - // which seems like it shouldn't change anything but does - // in fact require an increased path strip depth. - console.info('Retry due to miscalculated --strip-components depth'); - return await this.downloadAndExtract({ - withExtraStripDepth: 1, - url, - downloadTarget, - isCryptLibrary, - bindir, - }); - } - throw err; - } +async function lookupDownloadUrl({ + targetVersion, + enterprise, + options, +}: { + targetVersion: string; + enterprise: boolean; + options: DownloadOptions; +}): Promise { + return await getDownloadURL({ + version: targetVersion, + enterprise, + ...options, + }); +} + +export async function downloadMongoDb( + ...args: Parameters +): Promise { + return (await downloadMongoDbWithVersionInfo(...args)).downloadedBinDir; +} + +async function getCurrentDownloadedFile({ + bindir, + artifactInfoFile, +}: { + bindir: string; + artifactInfoFile: string; +}): Promise { + try { + await fs.stat(artifactInfoFile); + return { + ...JSON.parse(await fs.readFile(artifactInfoFile, 'utf8')), + downloadedBinDir: bindir, + }; + } catch { + /* ignore - file doesn't exist, proceed with download */ } } + +/** Runs the callback without a lock, using same interface as `withLock` */ +async function withoutLock( + bindir: string, + callback: () => Promise, +): Promise { + return await callback(); +} diff --git a/packages/mongodb-downloader/src/npm-with-lock.ts b/packages/mongodb-downloader/src/npm-with-lock.ts deleted file mode 100644 index 7f2afebb..00000000 --- a/packages/mongodb-downloader/src/npm-with-lock.ts +++ /dev/null @@ -1,406 +0,0 @@ -// Adapted from: -// https://raw.githubusercontent.com/npm/cli/072253549d774893a3689341dbc660cb845ebcfe/workspaces/libnpmexec/lib/with-lock.js - -// The Artistic License 2.0 -// Copyright (c) 2000-2006, The Perl Foundation. -// -// Everyone is permitted to copy and distribute verbatim copies -// of this license document, but changing it is not allowed. -// -// Preamble -// -// This license establishes the terms under which a given free software -// Package may be copied, modified, distributed, and/or redistributed. -// The intent is that the Copyright Holder maintains some artistic -// control over the development of that Package while still keeping the -// Package available as open source and free software. - -// You are always permitted to make arrangements wholly outside of this -// license directly with the Copyright Holder of a given Package. If the -// terms of this license do not permit the full use that you propose to -// make of the Package, you should contact the Copyright Holder and seek -// a different licensing arrangement. -// -// Definitions -// -// "Copyright Holder" means the individual(s) or organization(s) -// named in the copyright notice for the entire Package. -// -// "Contributor" means any party that has contributed code or other -// material to the Package, in accordance with the Copyright Holder's -// procedures. -// -// "You" and "your" means any person who would like to copy, -// distribute, or modify the Package. -// -// "Package" means the collection of files distributed by the -// Copyright Holder, and derivatives of that collection and/or of -// those files. A given Package may consist of either the Standard -// Version, or a Modified Version. -// -// "Distribute" means providing a copy of the Package or making it -// accessible to anyone else, or in the case of a company or -// organization, to others outside of your company or organization. -// -// "Distributor Fee" means any fee that you charge for Distributing -// this Package or providing support for this Package to another -// party. It does not mean licensing fees. -// -// "Standard Version" refers to the Package if it has not been -// modified, or has been modified only in ways explicitly requested -// by the Copyright Holder. -// -// "Modified Version" means the Package, if it has been changed, and -// such changes were not explicitly requested by the Copyright -// Holder. -// -// "Original License" means this Artistic License as Distributed with -// the Standard Version of the Package, in its current version or as -// it may be modified by The Perl Foundation in the future. -// -// "Source" form means the source code, documentation source, and -// configuration files for the Package. -// -// "Compiled" form means the compiled bytecode, object code, binary, -// or any other form resulting from mechanical transformation or -// translation of the Source form. -// -// Permission for Use and Modification Without Distribution - -// (1) You are permitted to use the Standard Version and create and use -// Modified Versions for any purpose without restriction, provided that -// you do not Distribute the Modified Version. -// -// Permissions for Redistribution of the Standard Version - -// (2) You may Distribute verbatim copies of the Source form of the -// Standard Version of this Package in any medium without restriction, -// either gratis or for a Distributor Fee, provided that you duplicate -// all of the original copyright notices and associated disclaimers. At -// your discretion, such verbatim copies may or may not include a -// Compiled form of the Package. - -// (3) You may apply any bug fixes, portability changes, and other -// modifications made available from the Copyright Holder. The resulting -// Package will still be considered the Standard Version, and as such -// will be subject to the Original License. -// -// Distribution of Modified Versions of the Package as Source -// -// (4) You may Distribute your Modified Version as Source (either gratis -// or for a Distributor Fee, and with or without a Compiled form of the -// Modified Version) provided that you clearly document how it differs -// from the Standard Version, including, but not limited to, documenting -// any non-standard features, executables, or modules, and provided that -// you do at least ONE of the following: -// -// (a) make the Modified Version available to the Copyright Holder -// of the Standard Version, under the Original License, so that the -// Copyright Holder may include your modifications in the Standard -// Version. -// -// (b) ensure that installation of your Modified Version does not -// prevent the user installing or running the Standard Version. In -// addition, the Modified Version must bear a name that is different -// from the name of the Standard Version. -// -// (c) allow anyone who receives a copy of the Modified Version to -// make the Source form of the Modified Version available to others -// under - -// (i) the Original License or -// -// (ii) a license that permits the licensee to freely copy, -// modify and redistribute the Modified Version using the same -// licensing terms that apply to the copy that the licensee -// received, and requires that the Source form of the Modified -// Version, and of any works derived from it, be made freely -// available in that license fees are prohibited but Distributor -// Fees are allowed. -// -// Distribution of Compiled Forms of the Standard Version -// or Modified Versions without the Source -// -// (5) You may Distribute Compiled forms of the Standard Version without -// the Source, provided that you include complete instructions on how to -// get the Source of the Standard Version. Such instructions must be -// valid at the time of your distribution. If these instructions, at any -// time while you are carrying out such distribution, become invalid, you -// must provide new instructions on demand or cease further distribution. -// If you provide valid instructions or cease distribution within thirty -// days after you become aware that the instructions are invalid, then -// you do not forfeit any of your rights under this license. -// -// (6) You may Distribute a Modified Version in Compiled form without -// the Source, provided that you comply with Section 4 with respect to -// the Source of the Modified Version. -// -// Aggregating or Linking the Package - -// (7) You may aggregate the Package (either the Standard Version or -// Modified Version) with other packages and Distribute the resulting -// aggregation provided that you do not charge a licensing fee for the -// Package. Distributor Fees are permitted, and licensing fees for other -// components in the aggregation are permitted. The terms of this license -// apply to the use and Distribution of the Standard or Modified Versions -// as included in the aggregation. -// -// (8) You are permitted to link Modified and Standard Versions with -// other works, to embed the Package in a larger work of your own, or to -// build stand-alone binary or bytecode versions of applications that -// include the Package, and Distribute the result without restriction, -// provided the result does not expose a direct interface to the Package. -// -// Items That are Not Considered Part of a Modified Version - -// (9) Works (including, but not limited to, modules and scripts) that -// merely extend or make use of the Package, do not, by themselves, cause -// the Package to be a Modified Version. In addition, such works are not -// considered parts of the Package itself, and are not subject to the -// terms of this license. -// -// General Provisions - -// (10) Any use, modification, and distribution of the Standard or -// Modified Versions is governed by this Artistic License. By using, -// modifying or distributing the Package, you accept this license. Do not -// use, modify, or distribute the Package, if you do not accept this -// license. -// -// (11) If your Modified Version has been derived from a Modified -// Version made by someone other than you, you are nevertheless required -// to ensure that your Modified Version complies with the requirements of -// this license. -// -// (12) This license does not grant you the right to use any trademark, -// service mark, tradename, or logo of the Copyright Holder. -// -// (13) This license includes the non-exclusive, worldwide, -// free-of-charge patent license to make, have made, use, offer to sell, -// sell, import and otherwise transfer the Package with respect to any -// patent claims licensable by the Copyright Holder that are necessarily -// infringed by the Package. If you institute patent litigation -// (including a cross-claim or counterclaim) against any party alleging -// that the Package constitutes direct or contributory patent -// infringement, then this Artistic License to you shall terminate on the -// date that such litigation is filed. -// -// (14) Disclaimer of Warranty: -// THE PACKAGE IS PROVIDED BY THE COPYRIGHT HOLDER AND CONTRIBUTORS "AS -// IS' AND WITHOUT ANY EXPRESS OR IMPLIED WARRANTIES. THE IMPLIED -// WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, OR -// NON-INFRINGEMENT ARE DISCLAIMED TO THE EXTENT PERMITTED BY YOUR LOCAL -// LAW. UNLESS REQUIRED BY LAW, NO COPYRIGHT HOLDER OR CONTRIBUTOR WILL -// BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL -// DAMAGES ARISING IN ANY WAY OUT OF THE USE OF THE PACKAGE, EVEN IF -// ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import fs from 'node:fs/promises'; -import { rmdirSync } from 'node:fs'; -import promiseRetry from 'promise-retry'; -import { onExit } from 'signal-exit'; - -// a lockfile implementation inspired by the unmaintained proper-lockfile library -// -// similarities: -// - based on mkdir's atomicity -// - works across processes and even machines (via NFS) -// - cleans up after itself -// - detects compromised locks -// -// differences: -// - higher-level API (just a withLock function) -// - written in async/await style -// - uses mtime + inode for more reliable compromised lock detection -// - more ergonomic compromised lock handling (i.e. withLock will reject, and callbacks have access to an AbortSignal) -// - uses a more recent version of signal-exit - -const touchInterval = 1_000; -// mtime precision is platform dependent, so use a reasonably large threshold -const staleThreshold = 5_000; - -// track current locks and their cleanup functions -const currentLocks = new Map void>(); - -function cleanupLocks() { - for (const [, cleanup] of currentLocks) { - try { - cleanup(); - } catch (err) { - // - } - } -} - -// clean up any locks that were not released normally -onExit(cleanupLocks); - -/** - * Acquire an advisory lock for the given path and hold it for the duration of the callback. - * - * The lock will be released automatically when the callback resolves or rejects. - * Concurrent calls to withLock() for the same path will wait until the lock is released. - */ -export async function withLock( - lockPath: string, - cb: (signal: AbortSignal) => Promise, -): Promise { - try { - const signal = await acquireLock(lockPath); - return await new Promise((resolve, reject) => { - signal.addEventListener('abort', () => { - reject( - Object.assign(new Error('Lock compromised'), { - code: 'ECOMPROMISED', - }), - ); - }); - - void (async () => { - try { - resolve(await cb(signal)); - } catch (err) { - reject(err); - } - })(); - }); - } finally { - releaseLock(lockPath); - } -} - -function acquireLock(lockPath: string): Promise { - return promiseRetry( - { - minTimeout: 100, - maxTimeout: 5_000, - // if another process legitimately holds the lock, wait for it to release; if it dies abnormally and the lock becomes stale, we'll acquire it automatically - forever: true, - }, - async (retry: (err: unknown) => never) => { - try { - await fs.mkdir(lockPath); - } catch (err: unknown) { - if ( - err && - typeof err === 'object' && - 'code' in err && - err.code !== 'EEXIST' && - err.code !== 'EBUSY' && - err.code !== 'EPERM' - ) { - throw err; - } - - const status = await getLockStatus(lockPath); - - if (status === 'locked') { - // let's see if we can acquire it on the next attempt 🤞 - return retry(err); - } - if (status === 'stale') { - try { - // there is a very tiny window where another process could also release the stale lock and acquire it before we release it here; the lock compromise checker should detect this and throw an error - deleteLock(lockPath); - } catch (e: unknown) { - // on windows, EBUSY/EPERM can happen if another process is (re)creating the lock; maybe we can acquire it on a subsequent attempt 🤞 - if ( - e && - typeof e === 'object' && - 'code' in e && - (e.code === 'EBUSY' || e.code === 'EPERM') - ) { - return retry(e); - } - throw e; - } - } - // immediately attempt to acquire the lock (no backoff) - return await acquireLock(lockPath); - } - try { - const signal = await maintainLock(lockPath); - return signal; - } catch (err) { - throw Object.assign(new Error('Lock compromised'), { - code: 'ECOMPROMISED', - }); - } - }, - ); -} - -function deleteLock(lockPath: string): void { - try { - // synchronous, so we can call in an exit handler - rmdirSync(lockPath); - } catch (err: unknown) { - if ( - err && - typeof err === 'object' && - 'code' in err && - err.code !== 'ENOENT' - ) { - throw err; - } - } -} - -function releaseLock(lockPath: string): void { - currentLocks.get(lockPath)?.(); - currentLocks.delete(lockPath); -} - -async function getLockStatus( - lockPath: string, -): Promise<'locked' | 'stale' | 'unlocked'> { - try { - const stat = await fs.stat(lockPath); - return Date.now() - stat.mtimeMs > staleThreshold ? 'stale' : 'locked'; - } catch (err: unknown) { - if ( - err && - typeof err === 'object' && - 'code' in err && - err.code === 'ENOENT' - ) { - return 'unlocked'; - } - throw err; - } -} - -async function maintainLock(lockPath: string): Promise { - const controller = new AbortController(); - const stats = await fs.stat(lockPath); - // fs.utimes operates on floating points seconds (directly, or via strings/Date objects), which may not match the underlying filesystem's mtime precision, meaning that we might read a slightly different mtime than we write. always round to the nearest second, since all filesystems support at least second precision - let mtime = Math.round(stats.mtimeMs / 1000); - const signal = controller.signal; - - async function touchLock() { - try { - const currentStats = await fs.stat(lockPath); - const currentMtime = Math.round(currentStats.mtimeMs / 1000); - if (currentStats.ino !== stats.ino || currentMtime !== mtime) { - throw new Error('Lock compromised'); - } - mtime = Math.round(Date.now() / 1000); - // touch the lock, unless we just released it during this iteration - if (currentLocks.has(lockPath)) { - await fs.utimes(lockPath, mtime, mtime); - } - } catch (err: unknown) { - // stats mismatch or other fs error means the lock was compromised - controller.abort(); - } - } - - const timeout = setInterval(() => void touchLock(), touchInterval); - timeout.unref(); - function cleanup() { - clearInterval(timeout); - deleteLock(lockPath); - } - currentLocks.set(lockPath, cleanup); - return signal; -} diff --git a/packages/mongodb-downloader/src/with-lock.ts b/packages/mongodb-downloader/src/with-lock.ts new file mode 100644 index 00000000..0e27e94e --- /dev/null +++ b/packages/mongodb-downloader/src/with-lock.ts @@ -0,0 +1,61 @@ +import lockfile from 'proper-lockfile'; +import fs from 'fs/promises'; + +/** + * Acquire an advisory lock for the given path and hold it for the duration of the callback. + * + * The lock will be released automatically when the callback resolves or rejects. + * Concurrent calls to withLock() for the same path will wait until the lock is released. + */ +export async function withLock( + originalFilePath: string, + cb: (signal: AbortSignal) => Promise, +): Promise { + const controller = new AbortController(); + let release: (() => Promise) | undefined; + + const lockFile = `${originalFilePath}/.mongodb-downloader-lock`; + + // Create the directory if it doesn't exist + await fs.mkdir(originalFilePath, { recursive: true }); + + // Create a dummy lock file to prevent concurrent downloads of the same version + await fs.writeFile(lockFile, ''); + + try { + release = await lockfile.lock(lockFile, { + retries: { + retries: 100, + minTimeout: 100, + maxTimeout: 5000, + factor: 2, + }, + stale: 100_000, + onCompromised: () => { + controller.abort(); + }, + }); + + return await new Promise((resolve, reject) => { + controller.signal.addEventListener('abort', () => { + reject(new Error('Aborted by abort signal')); + }); + + void (async () => { + try { + resolve(await cb(controller.signal)); + } catch (err) { + reject(err); + } + })(); + }); + } finally { + if (release) { + try { + await release(); + } catch (err) { + // Ignore errors during release + } + } + } +} From 8bab1484a1b950965818f9e36eb914e01a9a9893 Mon Sep 17 00:00:00 2001 From: gagik Date: Tue, 21 Oct 2025 10:32:46 +0200 Subject: [PATCH 03/12] chore: use class --- packages/mongodb-downloader/src/index.ts | 357 ++++++++++++----------- 1 file changed, 191 insertions(+), 166 deletions(-) diff --git a/packages/mongodb-downloader/src/index.ts b/packages/mongodb-downloader/src/index.ts index 20e72fb2..0584a967 100644 --- a/packages/mongodb-downloader/src/index.ts +++ b/packages/mongodb-downloader/src/index.ts @@ -33,193 +33,196 @@ export type MongoDBDownloaderOptions = { downloadOptions?: DownloadOptions; }; -// Download mongod + mongos and return the path to a directory containing them. -export async function downloadMongoDbWithVersionInfo({ - downloadOptions = {}, - version, - directory, - useLockfile, -}: MongoDBDownloaderOptions): Promise { - await fs.mkdir(directory, { recursive: true }); - const isWindows = ['win32', 'windows'].includes( - downloadOptions.platform ?? process.platform, - ); - const isCryptLibrary = !!downloadOptions.crypt_shared; - let isEnterprise = downloadOptions.enterprise ?? false; - let versionName = version; - - if (/-enterprise$/.test(version)) { - isEnterprise = true; - versionName = versionName.replace(/-enterprise$/, ''); - } +class MongoDBDownloader { + // Download mongod + mongos and return the path to a directory containing them. + async downloadMongoDbWithVersionInfo({ + downloadOptions = {}, + version, + directory, + useLockfile, + }: MongoDBDownloaderOptions): Promise { + await fs.mkdir(directory, { recursive: true }); + const isWindows = ['win32', 'windows'].includes( + downloadOptions.platform ?? process.platform, + ); + const isCryptLibrary = !!downloadOptions.crypt_shared; + let isEnterprise = downloadOptions.enterprise ?? false; + let versionName = version; - if (versionName !== 'latest-alpha') { - versionName = versionName + (isEnterprise ? '-enterprise' : '-community'); - } + if (/-enterprise$/.test(version)) { + isEnterprise = true; + versionName = versionName.replace(/-enterprise$/, ''); + } - const downloadTarget = path.resolve( - directory, - `mongodb-${process.platform}-${process.env.DISTRO_ID || 'none'}-${ - process.arch - }-${versionName}`.replace(/[^a-zA-Z0-9_-]/g, ''), - ); - return (async () => { + if (versionName !== 'latest-alpha') { + versionName = versionName + (isEnterprise ? '-enterprise' : '-community'); + } + + const downloadTarget = path.resolve( + directory, + `mongodb-${process.platform}-${process.env.DISTRO_ID || 'none'}-${ + process.arch + }-${versionName}`.replace(/[^a-zA-Z0-9_-]/g, ''), + ); const bindir = path.resolve( downloadTarget, isCryptLibrary && !isWindows ? 'lib' : 'bin', ); - const artifactInfoFile = path.join(bindir, '.artifact_info'); + return (async () => { + const artifactInfoFile = path.join(bindir, '.artifact_info'); - // Check if already downloaded before acquiring lock - const currentDownloadedFile = await getCurrentDownloadedFile({ - bindir, - artifactInfoFile, - }); - if (currentDownloadedFile) { - debug(`Skipping download because ${downloadTarget} exists`); - return currentDownloadedFile; - } + // Check if already downloaded before acquiring lock + const currentDownloadedFile = await this.getCurrentDownloadedFile({ + bindir, + artifactInfoFile, + }); + if (currentDownloadedFile) { + debug(`Skipping download because ${downloadTarget} exists`); + return currentDownloadedFile; + } - // Acquire the lock and perform download - return await (useLockfile ? withLock : withoutLock)( - downloadTarget, - async () => { - // Check again inside lock in case another process downloaded it - const downloadedFile = await getCurrentDownloadedFile({ - bindir, - artifactInfoFile, - }); - if (downloadedFile) { - debug( - `Skipping download because ${downloadTarget} exists (checked inside lock)`, - ); - return downloadedFile; - } - - await fs.mkdir(downloadTarget, { recursive: true }); - const artifactInfo = await lookupDownloadUrl({ - targetVersion: version, - enterprise: isEnterprise, - options: downloadOptions, - }); - const { url } = artifactInfo; - debug(`Downloading ${url} into ${downloadTarget}...`); + // Acquire the lock and perform download + return await (useLockfile ? withLock : withoutLock)( + downloadTarget, + async () => { + // Check again inside lock in case another process downloaded it + const downloadedFile = await this.getCurrentDownloadedFile({ + bindir, + artifactInfoFile, + }); + if (downloadedFile) { + debug( + `Skipping download because ${downloadTarget} exists after waiting on lock`, + ); + return downloadedFile; + } - await downloadAndExtract({ - url, - downloadTarget, - isCryptLibrary, - bindir, - }); - await fs.writeFile(artifactInfoFile, JSON.stringify(artifactInfo)); - debug(`Download complete`, bindir); - return { ...artifactInfo, downloadedBinDir: bindir }; - }, - ); - })(); -} + await fs.mkdir(downloadTarget, { recursive: true }); + const artifactInfo = await this.lookupDownloadUrl({ + targetVersion: version, + enterprise: isEnterprise, + options: downloadOptions, + }); + const { url } = artifactInfo; + debug(`Downloading ${url} into ${downloadTarget}...`); -const HWM = 1024 * 1024; - -async function downloadAndExtract({ - withExtraStripDepth = 0, - downloadTarget, - isCryptLibrary, - bindir, - url, -}: { - withExtraStripDepth?: number; - downloadTarget: string; - isCryptLibrary: boolean; - bindir: string; - url: string; -}): Promise { - const response = await fetch(url, { - highWaterMark: HWM, - } as Parameters[1]); - if (/\.tgz$|\.tar(\.[^.]+)?$/.exec(url)) { - // the server's tarballs can contain hard links, which the (unmaintained?) - // `download` package is unable to handle (https://github.com/kevva/decompress/issues/93) - await promisify(pipeline)( - response.body, - tar.x({ cwd: downloadTarget, strip: isCryptLibrary ? 0 : 1 }), - ); - } else { - const filename = path.join( - downloadTarget, - path.basename(new URL(url).pathname), - ); - await promisify(pipeline)( - response.body, - createWriteStream(filename, { highWaterMark: HWM }), - ); - debug(`Written file ${url} to ${filename}, extracting...`); - await decompress(filename, downloadTarget, { - strip: isCryptLibrary ? 0 : 1, - filter: (file) => path.extname(file.path) !== '.pdb', // Windows .pdb files are huge and useless - }); + await this.downloadAndExtract({ + url, + downloadTarget, + isCryptLibrary, + bindir, + }); + await fs.writeFile(artifactInfoFile, JSON.stringify(artifactInfo)); + debug(`Download complete`, bindir); + return { ...artifactInfo, downloadedBinDir: bindir }; + }, + ); + })(); } - try { - await fs.stat(bindir); // Make sure it exists. - } catch (err) { - if (withExtraStripDepth === 0 && url.includes('macos')) { - // The server team changed how macos release artifacts are packed - // and added a `./` prefix to paths in the tarball, - // which seems like it shouldn't change anything but does - // in fact require an increased path strip depth. - // eslint-disable-next-line no-console - console.info('Retry due to miscalculated --strip-components depth'); - return await downloadAndExtract({ - withExtraStripDepth: 1, - url, + static HWM = 1024 * 1024; + + private async downloadAndExtract({ + withExtraStripDepth = 0, + downloadTarget, + isCryptLibrary, + bindir, + url, + }: { + withExtraStripDepth?: number; + downloadTarget: string; + isCryptLibrary: boolean; + bindir: string; + url: string; + }): Promise { + const response = await fetch(url, { + highWaterMark: MongoDBDownloader.HWM, + } as Parameters[1]); + if (/\.tgz$|\.tar(\.[^.]+)?$/.exec(url)) { + // the server's tarballs can contain hard links, which the (unmaintained?) + // `download` package is unable to handle (https://github.com/kevva/decompress/issues/93) + await promisify(pipeline)( + response.body, + tar.x({ cwd: downloadTarget, strip: isCryptLibrary ? 0 : 1 }), + ); + } else { + const filename = path.join( downloadTarget, - isCryptLibrary, - bindir, + path.basename(new URL(url).pathname), + ); + await promisify(pipeline)( + response.body, + createWriteStream(filename, { highWaterMark: MongoDBDownloader.HWM }), + ); + debug(`Written file ${url} to ${filename}, extracting...`); + await decompress(filename, downloadTarget, { + strip: isCryptLibrary ? 0 : 1, + filter: (file) => path.extname(file.path) !== '.pdb', // Windows .pdb files are huge and useless }); } - throw err; + + try { + await fs.stat(bindir); // Make sure it exists. + } catch (err) { + if (withExtraStripDepth === 0 && url.includes('macos')) { + // The server team changed how macos release artifacts are packed + // and added a `./` prefix to paths in the tarball, + // which seems like it shouldn't change anything but does + // in fact require an increased path strip depth. + // eslint-disable-next-line no-console + console.info('Retry due to miscalculated --strip-components depth'); + return await this.downloadAndExtract({ + withExtraStripDepth: 1, + url, + downloadTarget, + isCryptLibrary, + bindir, + }); + } + throw err; + } } -} -async function lookupDownloadUrl({ - targetVersion, - enterprise, - options, -}: { - targetVersion: string; - enterprise: boolean; - options: DownloadOptions; -}): Promise { - return await getDownloadURL({ - version: targetVersion, + private async lookupDownloadUrl({ + targetVersion, enterprise, - ...options, - }); -} + options, + }: { + targetVersion: string; + enterprise: boolean; + options: DownloadOptions; + }): Promise { + return await getDownloadURL({ + version: targetVersion, + enterprise, + ...options, + }); + } -export async function downloadMongoDb( - ...args: Parameters -): Promise { - return (await downloadMongoDbWithVersionInfo(...args)).downloadedBinDir; -} + public async downloadMongoDb( + ...args: Parameters + ): Promise { + return (await this.downloadMongoDbWithVersionInfo(...args)) + .downloadedBinDir; + } -async function getCurrentDownloadedFile({ - bindir, - artifactInfoFile, -}: { - bindir: string; - artifactInfoFile: string; -}): Promise { - try { - await fs.stat(artifactInfoFile); - return { - ...JSON.parse(await fs.readFile(artifactInfoFile, 'utf8')), - downloadedBinDir: bindir, - }; - } catch { - /* ignore - file doesn't exist, proceed with download */ + private async getCurrentDownloadedFile({ + bindir, + artifactInfoFile, + }: { + bindir: string; + artifactInfoFile: string; + }): Promise { + try { + await fs.stat(artifactInfoFile); + return { + ...JSON.parse(await fs.readFile(artifactInfoFile, 'utf8')), + downloadedBinDir: bindir, + }; + } catch { + /* ignore - file doesn't exist, proceed with download */ + } } } @@ -230,3 +233,25 @@ async function withoutLock( ): Promise { return await callback(); } + +const downloader = new MongoDBDownloader(); + +export async function downloadMongoDbWithVersionInfo({ + downloadOptions = {}, + version, + directory, + useLockfile, +}: MongoDBDownloaderOptions): Promise { + return await downloader.downloadMongoDbWithVersionInfo({ + downloadOptions, + version, + directory, + useLockfile, + }); +} + +export async function downloadMongoDb( + ...args: Parameters +): Promise { + return await downloader.downloadMongoDb(...args); +} From 9f50210867e6c4b9ab8cb62f9f921e30f821f5cb Mon Sep 17 00:00:00 2001 From: gagik Date: Tue, 21 Oct 2025 10:49:17 +0200 Subject: [PATCH 04/12] chore: add proper tests --- package-lock.json | 10 + packages/mongodb-downloader/package.json | 5 + .../src/download-locking.spec.ts | 117 -------- packages/mongodb-downloader/src/index.ts | 6 +- .../mongodb-downloader/src/locking.spec.ts | 280 ++++++++++++++++++ 5 files changed, 298 insertions(+), 120 deletions(-) delete mode 100644 packages/mongodb-downloader/src/download-locking.spec.ts create mode 100644 packages/mongodb-downloader/src/locking.spec.ts diff --git a/package-lock.json b/package-lock.json index 34b91b03..3b55b9a6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -30742,6 +30742,7 @@ "@mongodb-js/mocha-config-devtools": "^1.0.5", "@mongodb-js/prettier-config-devtools": "^1.0.2", "@mongodb-js/tsconfig-devtools": "^1.0.3", + "@types/chai": "^4.2.21", "@types/debug": "^4.1.8", "@types/decompress": "^4.2.4", "@types/mocha": "^9.1.1", @@ -30749,13 +30750,17 @@ "@types/promise-retry": "^1.1.6", "@types/proper-lockfile": "^4.1.4", "@types/signal-exit": "^3.0.4", + "@types/sinon-chai": "^3.2.5", "@types/tar": "^6.1.5", + "chai": "^4.5.0", "depcheck": "^1.4.7", "eslint": "^7.25.0", "gen-esm-wrapper": "^1.1.3", "mocha": "^8.4.0", "nyc": "^15.1.0", "prettier": "^3.5.3", + "sinon": "^9.2.3", + "sinon-chai": "^3.7.0", "typescript": "^5.0.4" } }, @@ -38960,6 +38965,7 @@ "@mongodb-js/mocha-config-devtools": "^1.0.5", "@mongodb-js/prettier-config-devtools": "^1.0.2", "@mongodb-js/tsconfig-devtools": "^1.0.3", + "@types/chai": "^4.2.21", "@types/debug": "^4.1.8", "@types/decompress": "^4.2.4", "@types/mocha": "^9.1.1", @@ -38967,7 +38973,9 @@ "@types/promise-retry": "^1.1.6", "@types/proper-lockfile": "^4.1.4", "@types/signal-exit": "^3.0.4", + "@types/sinon-chai": "^3.2.5", "@types/tar": "^6.1.5", + "chai": "^4.5.0", "debug": "^4.4.0", "decompress": "^4.2.1", "depcheck": "^1.4.7", @@ -38981,6 +38989,8 @@ "promise-retry": "^2.0.1", "proper-lockfile": "^4.1.2", "signal-exit": "^4.1.0", + "sinon": "^9.2.3", + "sinon-chai": "^3.7.0", "tar": "^6.1.15", "typescript": "^5.0.4" }, diff --git a/packages/mongodb-downloader/package.json b/packages/mongodb-downloader/package.json index f94f5316..638d62c7 100644 --- a/packages/mongodb-downloader/package.json +++ b/packages/mongodb-downloader/package.json @@ -66,6 +66,7 @@ "@mongodb-js/mocha-config-devtools": "^1.0.5", "@mongodb-js/prettier-config-devtools": "^1.0.2", "@mongodb-js/tsconfig-devtools": "^1.0.3", + "@types/chai": "^4.2.21", "@types/debug": "^4.1.8", "@types/decompress": "^4.2.4", "@types/mocha": "^9.1.1", @@ -73,13 +74,17 @@ "@types/promise-retry": "^1.1.6", "@types/proper-lockfile": "^4.1.4", "@types/signal-exit": "^3.0.4", + "@types/sinon-chai": "^3.2.5", "@types/tar": "^6.1.5", + "chai": "^4.5.0", "depcheck": "^1.4.7", "eslint": "^7.25.0", "gen-esm-wrapper": "^1.1.3", "mocha": "^8.4.0", "nyc": "^15.1.0", "prettier": "^3.5.3", + "sinon": "^9.2.3", + "sinon-chai": "^3.7.0", "typescript": "^5.0.4" } } diff --git a/packages/mongodb-downloader/src/download-locking.spec.ts b/packages/mongodb-downloader/src/download-locking.spec.ts deleted file mode 100644 index c58c1173..00000000 --- a/packages/mongodb-downloader/src/download-locking.spec.ts +++ /dev/null @@ -1,117 +0,0 @@ -import { expect } from 'chai'; -import { promises as fs } from 'fs'; -import path from 'path'; -import os from 'os'; -import { downloadMongoDbWithVersionInfo } from '.'; - -describe('downloader with locking', function () { - this.timeout(60_000); - - let directory: string; - - beforeEach(async function () { - directory = path.join( - os.tmpdir(), - `download-integration-tests-${Date.now()}`, - ); - await fs.mkdir(directory, { recursive: true }); - }); - const version = '8.2.0'; - - afterEach(async function () { - try { - await fs.rm(directory, { recursive: true }); - } catch { - // Ignore cleanup errors - } - }); - - it('should prevent concurrent downloads of the same version', async function () { - const results = await Promise.all([ - downloadMongoDbWithVersionInfo({ directory, version, useLockfile: true }), - downloadMongoDbWithVersionInfo({ directory, version, useLockfile: true }), - downloadMongoDbWithVersionInfo({ directory, version, useLockfile: true }), - ]); - - // All results should be identical - expect(results[0].version).to.equal(version); - expect(results[1].version).to.equal(version); - expect(results[2].version).to.equal(version); - - expect(results[0].downloadedBinDir).to.equal(results[1].downloadedBinDir); - expect(results[1].downloadedBinDir).to.equal(results[2].downloadedBinDir); - - // Verify the downloaded directory exists and contains mongod - expect(await fs.stat(results[0].downloadedBinDir)).to.be.ok; - expect(await fs.stat(path.join(results[0].downloadedBinDir, 'mongod'))).to - .be.ok; - }); - - it('should wait for existing download to complete', async function () { - // First, download MongoDB normally - const result = await downloadMongoDbWithVersionInfo({ - directory, - version, - useLockfile: true, - }); - - expect(result.version).to.equal(version); - expect(result.downloadedBinDir).to.be.a('string'); - - // Verify the downloaded directory exists and contains mongod - expect(await fs.stat(result.downloadedBinDir)).to.be.ok; - expect(await fs.stat(path.join(result.downloadedBinDir, 'mongod'))).to.be - .ok; - }); - - it('should skip download if already completed', async function () { - // First download - const result1 = await downloadMongoDbWithVersionInfo({ - directory, - version, - useLockfile: true, - }); - - // Second download should use cached result - const result2 = await downloadMongoDbWithVersionInfo({ - directory, - version, - useLockfile: true, - }); - - expect(result1.version).to.equal(version); - expect(result2.version).to.equal(version); - expect(result1.downloadedBinDir).to.equal(result2.downloadedBinDir); - - // Verify the downloaded directory exists and contains mongod - expect(await fs.stat(result1.downloadedBinDir)).to.be.ok; - expect(await fs.stat(path.join(result1.downloadedBinDir, 'mongod'))).to.be - .ok; - }); - - it('should handle different versions independently', async function () { - const version2 = '8.1.0'; - - // Download different versions - const [result1, result2] = await Promise.all([ - downloadMongoDbWithVersionInfo({ directory, version, useLockfile: true }), - downloadMongoDbWithVersionInfo({ - directory, - version: version2, - useLockfile: true, - }), - ]); - - expect(result1.version).to.equal(version); - expect(result2.version).to.equal(version2); - expect(result1.downloadedBinDir).to.not.equal(result2.downloadedBinDir); - - // Verify both downloaded directories exist and contain mongod - expect(await fs.stat(result1.downloadedBinDir)).to.be.ok; - expect(await fs.stat(path.join(result1.downloadedBinDir, 'mongod'))).to.be - .ok; - expect(await fs.stat(result2.downloadedBinDir)).to.be.ok; - expect(await fs.stat(path.join(result2.downloadedBinDir, 'mongod'))).to.be - .ok; - }); -}); diff --git a/packages/mongodb-downloader/src/index.ts b/packages/mongodb-downloader/src/index.ts index 0584a967..53007efb 100644 --- a/packages/mongodb-downloader/src/index.ts +++ b/packages/mongodb-downloader/src/index.ts @@ -33,8 +33,7 @@ export type MongoDBDownloaderOptions = { downloadOptions?: DownloadOptions; }; -class MongoDBDownloader { - // Download mongod + mongos and return the path to a directory containing them. +export class MongoDBDownloader { async downloadMongoDbWithVersionInfo({ downloadOptions = {}, version, @@ -236,6 +235,7 @@ async function withoutLock( const downloader = new MongoDBDownloader(); +/** Download mongod + mongos with version info and return version info and the path to a directory containing them. */ export async function downloadMongoDbWithVersionInfo({ downloadOptions = {}, version, @@ -249,7 +249,7 @@ export async function downloadMongoDbWithVersionInfo({ useLockfile, }); } - +/** Download mongod + mongos and return the path to a directory containing them. */ export async function downloadMongoDb( ...args: Parameters ): Promise { diff --git a/packages/mongodb-downloader/src/locking.spec.ts b/packages/mongodb-downloader/src/locking.spec.ts new file mode 100644 index 00000000..32c85965 --- /dev/null +++ b/packages/mongodb-downloader/src/locking.spec.ts @@ -0,0 +1,280 @@ +import chai, { expect } from 'chai'; +import { promises as fs } from 'fs'; +import path from 'path'; +import os from 'os'; +import sinon from 'sinon'; +import sinonChai from 'sinon-chai'; +import { MongoDBDownloader } from '.'; + +chai.use(sinonChai); + +describe('using locks', function () { + this.timeout(60_000); + + let directory: string; + let testDownloader: MongoDBDownloader; + let downloadAndExtractStub: sinon.SinonStub; + let lookupDownloadUrlStub: sinon.SinonStub; + + beforeEach(async function () { + directory = path.join( + os.tmpdir(), + `download-integration-tests-${Date.now()}`, + ); + await fs.mkdir(directory, { recursive: true }); + + // Create a test instance of the downloader + testDownloader = new MongoDBDownloader(); + + // Mock the downloadAndExtract method to avoid actual downloads + // eslint-disable-next-line @typescript-eslint/no-explicit-any + downloadAndExtractStub = sinon + .stub(testDownloader as any, 'downloadAndExtract') + .callsFake(async (...args: any[]) => { + // Create the bindir and a fake mongod executable + const params = args[0] as { + bindir: string; + url: string; + downloadTarget: string; + isCryptLibrary: boolean; + }; + await fs.mkdir(params.bindir, { recursive: true }); + await fs.writeFile( + path.join(params.bindir, 'mongod'), + '#!/bin/bash\necho "This is a mock mongod"', + ); + }); + + // Mock the lookupDownloadUrl method to avoid network calls + lookupDownloadUrlStub = sinon + .stub(testDownloader as any, 'lookupDownloadUrl') + .resolves({ + version: '8.2.0', + url: 'https://example.com/mongodb-8.2.0.tgz', + name: 'mongodb-8.2.0', + }); + }); + const version = '8.2.0'; + + afterEach(async function () { + // Restore stubs + downloadAndExtractStub.restore(); + lookupDownloadUrlStub.restore(); + + try { + await fs.rm(directory, { recursive: true }); + } catch { + // Ignore cleanup errors + } + }); + + describe('without lockfile', function () { + it('should download multiple times in parallel', async function () { + const results = await Promise.all([ + testDownloader.downloadMongoDbWithVersionInfo({ + directory, + version, + useLockfile: false, + }), + testDownloader.downloadMongoDbWithVersionInfo({ + directory, + version, + useLockfile: false, + }), + testDownloader.downloadMongoDbWithVersionInfo({ + directory, + version, + useLockfile: false, + }), + ]); + expect(results[0].version).to.equal(version); + expect(results[0].downloadedBinDir).to.be.a('string'); + + expect(downloadAndExtractStub).to.have.callCount(3); + }); + + it('should skip download if already completed sequentially', async function () { + const result = await testDownloader.downloadMongoDbWithVersionInfo({ + directory, + version, + useLockfile: false, + }); + + const result2 = await testDownloader.downloadMongoDbWithVersionInfo({ + directory, + version, + useLockfile: false, + }); + expect(result2.version).to.equal(version); + expect(result2.downloadedBinDir).to.equal(result.downloadedBinDir); + + expect(downloadAndExtractStub).to.have.been.calledOnce; + }); + + it('should handle different versions independently', async function () { + const version2 = '8.1.0'; + + // Update stub to return different version info for the second call + lookupDownloadUrlStub.onSecondCall().resolves({ + version: '8.1.0', + url: 'https://example.com/mongodb-8.1.0.tgz', + name: 'mongodb-8.1.0', + }); + + // Download different versions + const [result1, result2] = await Promise.all([ + testDownloader.downloadMongoDbWithVersionInfo({ + directory, + version, + useLockfile: false, + }), + testDownloader.downloadMongoDbWithVersionInfo({ + directory, + version: version2, + useLockfile: false, + }), + ]); + + expect(result1.version).to.equal(version); + expect(result2.version).to.equal(version2); + expect(result1.downloadedBinDir).to.not.equal(result2.downloadedBinDir); + + // Verify both downloaded directories exist and contain mongod + expect(await fs.stat(result1.downloadedBinDir)).to.be.ok; + expect(await fs.stat(path.join(result1.downloadedBinDir, 'mongod'))).to.be + .ok; + expect(await fs.stat(result2.downloadedBinDir)).to.be.ok; + expect(await fs.stat(path.join(result2.downloadedBinDir, 'mongod'))).to.be + .ok; + + // Verify downloadAndExtract was called twice (once for each version) + expect(downloadAndExtractStub).to.have.been.calledTwice; + }); + }); + + describe('with lockfile', function () { + it('should prevent concurrent downloads of the same version', async function () { + const results = await Promise.all([ + testDownloader.downloadMongoDbWithVersionInfo({ + directory, + version, + useLockfile: true, + }), + testDownloader.downloadMongoDbWithVersionInfo({ + directory, + version, + useLockfile: true, + }), + testDownloader.downloadMongoDbWithVersionInfo({ + directory, + version, + useLockfile: true, + }), + ]); + + // All results should be identical + expect(results[0].version).to.equal(version); + expect(results[1].version).to.equal(version); + expect(results[2].version).to.equal(version); + + expect(results[0].downloadedBinDir).to.equal(results[1].downloadedBinDir); + expect(results[1].downloadedBinDir).to.equal(results[2].downloadedBinDir); + + // Verify the downloaded directory exists and contains mongod + expect(await fs.stat(results[0].downloadedBinDir)).to.be.ok; + expect(await fs.stat(path.join(results[0].downloadedBinDir, 'mongod'))).to + .be.ok; + + // Verify downloadAndExtract was called only once despite 3 concurrent requests + expect(downloadAndExtractStub).to.have.been.calledOnce; + }); + + it('should wait for existing download to complete', async function () { + // First, download MongoDB normally + const result = await testDownloader.downloadMongoDbWithVersionInfo({ + directory, + version, + useLockfile: true, + }); + + expect(result.version).to.equal(version); + expect(result.downloadedBinDir).to.be.a('string'); + + // Verify the downloaded directory exists and contains mongod + expect(await fs.stat(result.downloadedBinDir)).to.be.ok; + expect(await fs.stat(path.join(result.downloadedBinDir, 'mongod'))).to.be + .ok; + + // Verify downloadAndExtract was called once + expect(downloadAndExtractStub).to.have.been.calledOnce; + }); + + it('should skip download if already completed', async function () { + // First download + const result1 = await testDownloader.downloadMongoDbWithVersionInfo({ + directory, + version, + useLockfile: true, + }); + + // Second download should use cached result + const result2 = await testDownloader.downloadMongoDbWithVersionInfo({ + directory, + version, + useLockfile: true, + }); + + expect(result1.version).to.equal(version); + expect(result2.version).to.equal(version); + expect(result1.downloadedBinDir).to.equal(result2.downloadedBinDir); + + // Verify the downloaded directory exists and contains mongod + expect(await fs.stat(result1.downloadedBinDir)).to.be.ok; + expect(await fs.stat(path.join(result1.downloadedBinDir, 'mongod'))).to.be + .ok; + + // Verify downloadAndExtract was called only once, not twice + expect(downloadAndExtractStub).to.have.been.calledOnce; + }); + + it('should handle different versions independently', async function () { + const version2 = '8.1.0'; + + // Update stub to return different version info for the second call + lookupDownloadUrlStub.onSecondCall().resolves({ + version: '8.1.0', + url: 'https://example.com/mongodb-8.1.0.tgz', + name: 'mongodb-8.1.0', + }); + + // Download different versions + const [result1, result2] = await Promise.all([ + testDownloader.downloadMongoDbWithVersionInfo({ + directory, + version, + useLockfile: true, + }), + testDownloader.downloadMongoDbWithVersionInfo({ + directory, + version: version2, + useLockfile: true, + }), + ]); + + expect(result1.version).to.equal(version); + expect(result2.version).to.equal(version2); + expect(result1.downloadedBinDir).to.not.equal(result2.downloadedBinDir); + + // Verify both downloaded directories exist and contain mongod + expect(await fs.stat(result1.downloadedBinDir)).to.be.ok; + expect(await fs.stat(path.join(result1.downloadedBinDir, 'mongod'))).to.be + .ok; + expect(await fs.stat(result2.downloadedBinDir)).to.be.ok; + expect(await fs.stat(path.join(result2.downloadedBinDir, 'mongod'))).to.be + .ok; + + // Verify downloadAndExtract was called twice (once for each version) + expect(downloadAndExtractStub).to.have.been.calledTwice; + }); + }); +}); From fb9f1c73bc4ff5db808abab7d10143bd40bf610d Mon Sep 17 00:00:00 2001 From: gagik Date: Tue, 21 Oct 2025 11:09:16 +0200 Subject: [PATCH 05/12] chore: version naming tests --- .../mongodb-downloader/src/locking.spec.ts | 124 +++++++++++++++++- 1 file changed, 118 insertions(+), 6 deletions(-) diff --git a/packages/mongodb-downloader/src/locking.spec.ts b/packages/mongodb-downloader/src/locking.spec.ts index 32c85965..62a58ee5 100644 --- a/packages/mongodb-downloader/src/locking.spec.ts +++ b/packages/mongodb-downloader/src/locking.spec.ts @@ -8,7 +8,7 @@ import { MongoDBDownloader } from '.'; chai.use(sinonChai); -describe('using locks', function () { +describe('MongoDBDownloader', function () { this.timeout(60_000); let directory: string; @@ -135,8 +135,7 @@ describe('using locks', function () { }), ]); - expect(result1.version).to.equal(version); - expect(result2.version).to.equal(version2); + expect(result1.version).to.not.equal(result2.version); expect(result1.downloadedBinDir).to.not.equal(result2.downloadedBinDir); // Verify both downloaded directories exist and contain mongod @@ -226,7 +225,6 @@ describe('using locks', function () { expect(result1.version).to.equal(version); expect(result2.version).to.equal(version); - expect(result1.downloadedBinDir).to.equal(result2.downloadedBinDir); // Verify the downloaded directory exists and contains mongod expect(await fs.stat(result1.downloadedBinDir)).to.be.ok; @@ -261,8 +259,7 @@ describe('using locks', function () { }), ]); - expect(result1.version).to.equal(version); - expect(result2.version).to.equal(version2); + expect(result1.version).to.not.equal(result2.version); expect(result1.downloadedBinDir).to.not.equal(result2.downloadedBinDir); // Verify both downloaded directories exist and contain mongod @@ -277,4 +274,119 @@ describe('using locks', function () { expect(downloadAndExtractStub).to.have.been.calledTwice; }); }); + + describe('version name', function () { + for (const { + version, + enterprise, + expectedVersion, + expectedVersionName, + expectedEnterpriseFlag, + } of [ + { + version: '8.1.0', + enterprise: undefined, + expectedVersion: '8.1.0', + expectedVersionName: '8.1.0-community', + expectedEnterpriseFlag: false, + }, + { + version: '8.1.0', + enterprise: false, + expectedVersion: '8.1.0', + expectedVersionName: '8.1.0-community', + expectedEnterpriseFlag: false, + }, + { + version: '8.1.0', + enterprise: true, + expectedVersion: '8.1.0', + expectedVersionName: '8.1.0-enterprise', + expectedEnterpriseFlag: true, + }, + { + version: '8.1.0-enterprise', + enterprise: undefined, + expectedVersion: '8.1.0-enterprise', + expectedVersionName: '8.1.0', + expectedEnterpriseFlag: true, + }, + { + version: '8.1.0-enterprise', + enterprise: false, + expectedVersion: '8.1.0-enterprise', + expectedVersionName: '8.1.0-enterprise', + expectedEnterpriseFlag: true, + }, + { + version: '8.1.0-enterprise', + enterprise: true, + expectedVersion: '8.1.0-enterprise', + expectedVersionName: '8.1.0-enterprise', + expectedEnterpriseFlag: true, + }, + { + version: 'latest-alpha', + enterprise: undefined, + expectedVersion: 'latest-alpha', + expectedVersionName: 'latest-alpha', + expectedEnterpriseFlag: false, + }, + { + version: 'latest-alpha', + enterprise: true, + expectedVersion: 'latest-alpha', + expectedVersionName: 'latest-alpha', + expectedEnterpriseFlag: true, + }, + { + version: '7.0.5', + enterprise: false, + expectedVersion: '7.0.5', + expectedVersionName: '7.0.5-community', + expectedEnterpriseFlag: false, + }, + { + version: '8.1.0-rc0', + enterprise: false, + expectedVersion: '8.1.0-rc0', + expectedVersionName: '8.1.0-rc0-community', + expectedEnterpriseFlag: false, + }, + ]) { + it(`should resolve correct version for ${version} with enterprise=${String(enterprise)}`, async function () { + lookupDownloadUrlStub.resetHistory(); + + const opts: { + directory: string; + version: string; + useLockfile: boolean; + downloadOptions?: { enterprise: boolean }; + } = { + directory, + version, + useLockfile: false, + }; + + if (enterprise !== undefined) { + opts.downloadOptions = { enterprise: enterprise }; + } + + const result = + await testDownloader.downloadMongoDbWithVersionInfo(opts); + + // Verify lookup call + expect(lookupDownloadUrlStub).to.have.been.calledOnce; + + const callArgs = lookupDownloadUrlStub.firstCall.args[0]; + expect(callArgs.targetVersion).to.equal(expectedVersion); + expect(callArgs.enterprise).to.equal(expectedEnterpriseFlag); + + // Verify path contains expected string + expect(result.downloadedBinDir).to.include( + expectedVersionName.replaceAll('.', ''), + ); + }); + } + }); }); From e14716ad5220ff2f60e2437e7dc6ca8f8db82548 Mon Sep 17 00:00:00 2001 From: gagik Date: Tue, 21 Oct 2025 11:10:25 +0200 Subject: [PATCH 06/12] chore: rename file --- .../mongodb-downloader/src/{locking.spec.ts => index.spec.ts} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename packages/mongodb-downloader/src/{locking.spec.ts => index.spec.ts} (100%) diff --git a/packages/mongodb-downloader/src/locking.spec.ts b/packages/mongodb-downloader/src/index.spec.ts similarity index 100% rename from packages/mongodb-downloader/src/locking.spec.ts rename to packages/mongodb-downloader/src/index.spec.ts From 365a452b912e5af38176335e40be51613a13d05a Mon Sep 17 00:00:00 2001 From: gagik Date: Tue, 21 Oct 2025 14:45:50 +0200 Subject: [PATCH 07/12] docs: add migration guide --- packages/mongodb-downloader/README.md | 39 +++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 packages/mongodb-downloader/README.md diff --git a/packages/mongodb-downloader/README.md b/packages/mongodb-downloader/README.md new file mode 100644 index 00000000..96048173 --- /dev/null +++ b/packages/mongodb-downloader/README.md @@ -0,0 +1,39 @@ +## mongodb-downloader + +A simple library to download MongoDB binaries for different platforms and versions. + +### Migrating from v0.4 to v0.5 + +In v0.5.x, the library introduced lockfiles to prevent parallel downloads of the same MongoDB binary. It also changed the arguments for the `downloadMongoDb` and `downloadMongoDbWithVersionInfo` functions, introducing a new `useLockfile` field and using a single options object instead of separate parameters for different options. It is recommended to enable lockfiles to prevent redundant downloads unless you have a specific reason not to. + +```ts +// Before (v0.4.x) +downloadMongoDb('/tmp/directory', '4.4.6', { + platform: 'linux', + arch: 'x64', +}); + +downloadMongoDbWithVersionInfo('/tmp/directory', '4.4.6', { + arch: 'x64', +}); + +// After (v0.5.x) +downloadMongoDb({ + downloadDir: '/tmp/directory', + version: '4.4.6', + useLockfile: true, // New, required field. + downloadOptions: { + platform: 'linux', + arch: 'x64', + }, +}); + +downloadMongoDbWithVersionInfo({ + downloadDir: '/tmp/directory', + version: '4.4.6', + useLockfile: true, // New, required field. + downloadOptions: { + arch: 'x64', + }, +}); +``` From 6e0975e2879702b11c894374340f48945f36e70d Mon Sep 17 00:00:00 2001 From: gagik Date: Tue, 21 Oct 2025 15:23:49 +0200 Subject: [PATCH 08/12] chore: correct docs --- packages/mongodb-downloader/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/mongodb-downloader/README.md b/packages/mongodb-downloader/README.md index 96048173..d853d1e7 100644 --- a/packages/mongodb-downloader/README.md +++ b/packages/mongodb-downloader/README.md @@ -19,7 +19,7 @@ downloadMongoDbWithVersionInfo('/tmp/directory', '4.4.6', { // After (v0.5.x) downloadMongoDb({ - downloadDir: '/tmp/directory', + directory: '/tmp/directory', version: '4.4.6', useLockfile: true, // New, required field. downloadOptions: { @@ -29,7 +29,7 @@ downloadMongoDb({ }); downloadMongoDbWithVersionInfo({ - downloadDir: '/tmp/directory', + directory: '/tmp/directory', version: '4.4.6', useLockfile: true, // New, required field. downloadOptions: { From 7078c3f1a043d4a93799c09633d1a69b9a7c550e Mon Sep 17 00:00:00 2001 From: gagik Date: Wed, 22 Oct 2025 16:44:49 +0200 Subject: [PATCH 09/12] chore: fix package.json --- package-lock.json | 240 +++++++++++++++++------ packages/mongodb-downloader/package.json | 10 +- 2 files changed, 185 insertions(+), 65 deletions(-) diff --git a/package-lock.json b/package-lock.json index 3b55b9a6..b996f795 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10367,16 +10367,6 @@ "integrity": "sha512-+68kP9yzs4LMp7VNh8gdzMSPZFL44MLGqiHWvttYJe+6qnuVr4Ek9wSBQoveqY/r+LwjCcU29kNVkidwim+kYA==", "dev": true }, - "node_modules/@types/promise-retry": { - "version": "1.1.6", - "resolved": "https://registry.npmjs.org/@types/promise-retry/-/promise-retry-1.1.6.tgz", - "integrity": "sha512-EC1+OMXV0PZb0pf+cmyxc43MEP2CDumZe4AfuxWboxxEixztIebknpJPZAX5XlodGF1OY+C1E/RAeNGzxf+bJA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/retry": "*" - } - }, "node_modules/@types/prop-types": { "version": "15.7.11", "resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.11.tgz", @@ -10474,13 +10464,6 @@ "integrity": "sha512-k107IF4+Xr7UHjwDc7Cfd6PRQfbdkiRabXGRjo07b4WyPahFBZCZ1sE+BNxYIJPPg73UkfOsVOLwqVc/6ETrIA==", "license": "MIT" }, - "node_modules/@types/signal-exit": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/@types/signal-exit/-/signal-exit-3.0.4.tgz", - "integrity": "sha512-e7EUPfU9afHyWc5CXtlqbvVHEshrb05uPlDCenWIbMgtWoFrTuTDVYNLKk6o4X2/4oHTfNqrJX/vaJ3uBhtXTg==", - "dev": true, - "license": "MIT" - }, "node_modules/@types/sinon": { "version": "9.0.11", "resolved": "https://registry.npmjs.org/@types/sinon/-/sinon-9.0.11.tgz", @@ -30725,16 +30708,14 @@ }, "packages/mongodb-downloader": { "name": "@mongodb-js/mongodb-downloader", - "version": "0.4.3", + "version": "0.5.0", "license": "Apache-2.0", "dependencies": { "debug": "^4.4.0", "decompress": "^4.2.1", "mongodb-download-url": "^1.6.3", "node-fetch": "^2.7.0", - "promise-retry": "^2.0.1", "proper-lockfile": "^4.1.2", - "signal-exit": "^4.1.0", "tar": "^6.1.15" }, "devDependencies": { @@ -30747,9 +30728,7 @@ "@types/decompress": "^4.2.4", "@types/mocha": "^9.1.1", "@types/node": "^22.15.30", - "@types/promise-retry": "^1.1.6", "@types/proper-lockfile": "^4.1.4", - "@types/signal-exit": "^3.0.4", "@types/sinon-chai": "^3.2.5", "@types/tar": "^6.1.5", "chai": "^4.5.0", @@ -30792,18 +30771,6 @@ } } }, - "packages/mongodb-downloader/node_modules/signal-exit": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", - "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", - "license": "ISC", - "engines": { - "node": ">=14" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, "packages/mongodb-downloader/node_modules/tar": { "version": "6.2.1", "resolved": "https://registry.npmjs.org/tar/-/tar-6.2.1.tgz", @@ -31022,6 +30989,19 @@ "typescript": "^5.0.4" } }, + "packages/mongodb-runner/node_modules/@mongodb-js/mongodb-downloader": { + "version": "0.4.4", + "resolved": "https://registry.npmjs.org/@mongodb-js/mongodb-downloader/-/mongodb-downloader-0.4.4.tgz", + "integrity": "sha512-jzRnA8GHvHRXJHUKqfcHefKyDH1W3u6z0MTBb21nuGmAdNaVB3588eL8thieivUIpVTDWRPF54deHLtoHUrIGg==", + "license": "Apache-2.0", + "dependencies": { + "debug": "^4.4.0", + "decompress": "^4.2.1", + "mongodb-download-url": "^1.6.4", + "node-fetch": "^2.7.0", + "tar": "^6.1.15" + } + }, "packages/mongodb-runner/node_modules/cliui": { "version": "8.0.1", "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", @@ -31035,6 +31015,85 @@ "node": ">=12" } }, + "packages/mongodb-runner/node_modules/minipass": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-5.0.0.tgz", + "integrity": "sha512-3FnjYuehv9k6ovOEbyOswadCDPX1piCfhV8ncmYtHOjuPwylVWsghTLo7rabjC3Rx5xD4HDx8Wm1xnMF7S5qFQ==", + "license": "ISC", + "engines": { + "node": ">=8" + } + }, + "packages/mongodb-runner/node_modules/mongodb-download-url": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/mongodb-download-url/-/mongodb-download-url-1.7.0.tgz", + "integrity": "sha512-Dj9l3/MzvgAO+no7zaBIbQL0Ilsb0jo5Y8WijkB/sCnagoh3KgnN0Vxu5awglK/75QrEObduf9wa+U0fjxvYMw==", + "license": "Apache-2.0", + "dependencies": { + "debug": "^4.4.0", + "minimist": "^1.2.8", + "node-fetch": "^2.7.0", + "semver": "^7.7.1" + }, + "bin": { + "mongodb-download-url": "bin/mongodb-download-url.js" + } + }, + "packages/mongodb-runner/node_modules/node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "license": "MIT", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, + "packages/mongodb-runner/node_modules/semver": { + "version": "7.7.3", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.3.tgz", + "integrity": "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "packages/mongodb-runner/node_modules/tar": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/tar/-/tar-6.2.1.tgz", + "integrity": "sha512-DZ4yORTwrbTj/7MZYq2w+/ZFdI6OZ/f9SFHR+71gIVUZhOQPHzVCLpvRnPgyaMpfWxxk/4ONva3GQSyNIKRv6A==", + "license": "ISC", + "dependencies": { + "chownr": "^2.0.0", + "fs-minipass": "^2.0.0", + "minipass": "^5.0.0", + "minizlib": "^2.1.1", + "mkdirp": "^1.0.3", + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "packages/mongodb-runner/node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", + "license": "MIT" + }, "packages/mongodb-runner/node_modules/typescript": { "version": "5.8.2", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.8.2.tgz", @@ -31049,6 +31108,22 @@ "node": ">=14.17" } }, + "packages/mongodb-runner/node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", + "license": "BSD-2-Clause" + }, + "packages/mongodb-runner/node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "license": "MIT", + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, "packages/mongodb-runner/node_modules/yargs": { "version": "17.7.2", "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", @@ -38970,9 +39045,7 @@ "@types/decompress": "^4.2.4", "@types/mocha": "^9.1.1", "@types/node": "^22.15.30", - "@types/promise-retry": "^1.1.6", "@types/proper-lockfile": "^4.1.4", - "@types/signal-exit": "^3.0.4", "@types/sinon-chai": "^3.2.5", "@types/tar": "^6.1.5", "chai": "^4.5.0", @@ -38986,9 +39059,7 @@ "node-fetch": "^2.7.0", "nyc": "^15.1.0", "prettier": "^3.5.3", - "promise-retry": "^2.0.1", "proper-lockfile": "^4.1.2", - "signal-exit": "^4.1.0", "sinon": "^9.2.3", "sinon-chai": "^3.7.0", "tar": "^6.1.15", @@ -39008,11 +39079,6 @@ "whatwg-url": "^5.0.0" } }, - "signal-exit": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", - "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==" - }, "tar": { "version": "6.2.1", "resolved": "https://registry.npmjs.org/tar/-/tar-6.2.1.tgz", @@ -41784,15 +41850,6 @@ "integrity": "sha512-+68kP9yzs4LMp7VNh8gdzMSPZFL44MLGqiHWvttYJe+6qnuVr4Ek9wSBQoveqY/r+LwjCcU29kNVkidwim+kYA==", "dev": true }, - "@types/promise-retry": { - "version": "1.1.6", - "resolved": "https://registry.npmjs.org/@types/promise-retry/-/promise-retry-1.1.6.tgz", - "integrity": "sha512-EC1+OMXV0PZb0pf+cmyxc43MEP2CDumZe4AfuxWboxxEixztIebknpJPZAX5XlodGF1OY+C1E/RAeNGzxf+bJA==", - "dev": true, - "requires": { - "@types/retry": "*" - } - }, "@types/prop-types": { "version": "15.7.11", "resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.11.tgz", @@ -41886,12 +41943,6 @@ "resolved": "https://registry.npmjs.org/@types/semver/-/semver-7.7.0.tgz", "integrity": "sha512-k107IF4+Xr7UHjwDc7Cfd6PRQfbdkiRabXGRjo07b4WyPahFBZCZ1sE+BNxYIJPPg73UkfOsVOLwqVc/6ETrIA==" }, - "@types/signal-exit": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/@types/signal-exit/-/signal-exit-3.0.4.tgz", - "integrity": "sha512-e7EUPfU9afHyWc5CXtlqbvVHEshrb05uPlDCenWIbMgtWoFrTuTDVYNLKk6o4X2/4oHTfNqrJX/vaJ3uBhtXTg==", - "dev": true - }, "@types/sinon": { "version": "9.0.11", "resolved": "https://registry.npmjs.org/@types/sinon/-/sinon-9.0.11.tgz", @@ -51080,6 +51131,18 @@ "yargs": "^17.7.2" }, "dependencies": { + "@mongodb-js/mongodb-downloader": { + "version": "0.4.4", + "resolved": "https://registry.npmjs.org/@mongodb-js/mongodb-downloader/-/mongodb-downloader-0.4.4.tgz", + "integrity": "sha512-jzRnA8GHvHRXJHUKqfcHefKyDH1W3u6z0MTBb21nuGmAdNaVB3588eL8thieivUIpVTDWRPF54deHLtoHUrIGg==", + "requires": { + "debug": "^4.4.0", + "decompress": "^4.2.1", + "mongodb-download-url": "^1.6.4", + "node-fetch": "^2.7.0", + "tar": "^6.1.15" + } + }, "cliui": { "version": "8.0.1", "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", @@ -51090,12 +51153,73 @@ "wrap-ansi": "^7.0.0" } }, + "minipass": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-5.0.0.tgz", + "integrity": "sha512-3FnjYuehv9k6ovOEbyOswadCDPX1piCfhV8ncmYtHOjuPwylVWsghTLo7rabjC3Rx5xD4HDx8Wm1xnMF7S5qFQ==" + }, + "mongodb-download-url": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/mongodb-download-url/-/mongodb-download-url-1.7.0.tgz", + "integrity": "sha512-Dj9l3/MzvgAO+no7zaBIbQL0Ilsb0jo5Y8WijkB/sCnagoh3KgnN0Vxu5awglK/75QrEObduf9wa+U0fjxvYMw==", + "requires": { + "debug": "^4.4.0", + "minimist": "^1.2.8", + "node-fetch": "^2.7.0", + "semver": "^7.7.1" + } + }, + "node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "requires": { + "whatwg-url": "^5.0.0" + } + }, + "semver": { + "version": "7.7.3", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.3.tgz", + "integrity": "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q==" + }, + "tar": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/tar/-/tar-6.2.1.tgz", + "integrity": "sha512-DZ4yORTwrbTj/7MZYq2w+/ZFdI6OZ/f9SFHR+71gIVUZhOQPHzVCLpvRnPgyaMpfWxxk/4ONva3GQSyNIKRv6A==", + "requires": { + "chownr": "^2.0.0", + "fs-minipass": "^2.0.0", + "minipass": "^5.0.0", + "minizlib": "^2.1.1", + "mkdirp": "^1.0.3", + "yallist": "^4.0.0" + } + }, + "tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" + }, "typescript": { "version": "5.8.2", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.8.2.tgz", "integrity": "sha512-aJn6wq13/afZp/jT9QZmwEjDqqvSGp1VT5GVg+f/t6/oVyrgXM6BY1h9BRh/O5p3PlUPAe+WuiEZOmb/49RqoQ==", "dev": true }, + "webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" + }, + "whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "requires": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, "yargs": { "version": "17.7.2", "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", diff --git a/packages/mongodb-downloader/package.json b/packages/mongodb-downloader/package.json index 638d62c7..2704eee4 100644 --- a/packages/mongodb-downloader/package.json +++ b/packages/mongodb-downloader/package.json @@ -13,7 +13,7 @@ "email": "compass@mongodb.com" }, "homepage": "https://github.com/mongodb-js/devtools-shared", - "version": "0.4.3", + "version": "0.5.0", "repository": { "type": "git", "url": "https://github.com/mongodb-js/devtools-shared.git" @@ -53,13 +53,11 @@ }, "dependencies": { "debug": "^4.4.0", + "tar": "^6.1.15", "decompress": "^4.2.1", "mongodb-download-url": "^1.6.3", "node-fetch": "^2.7.0", - "promise-retry": "^2.0.1", - "proper-lockfile": "^4.1.2", - "signal-exit": "^4.1.0", - "tar": "^6.1.15" + "proper-lockfile": "^4.1.2" }, "devDependencies": { "@mongodb-js/eslint-config-devtools": "0.9.12", @@ -71,9 +69,7 @@ "@types/decompress": "^4.2.4", "@types/mocha": "^9.1.1", "@types/node": "^22.15.30", - "@types/promise-retry": "^1.1.6", "@types/proper-lockfile": "^4.1.4", - "@types/signal-exit": "^3.0.4", "@types/sinon-chai": "^3.2.5", "@types/tar": "^6.1.5", "chai": "^4.5.0", From ea6d5c073a50566634de7123d5792724b33f67ed Mon Sep 17 00:00:00 2001 From: gagik Date: Wed, 22 Oct 2025 21:10:06 +0200 Subject: [PATCH 10/12] chore: update mongodb-runner --- packages/mongodb-downloader/src/index.ts | 31 +++++++++++---------- packages/mongodb-runner/src/mongocluster.ts | 7 ++++- 2 files changed, 23 insertions(+), 15 deletions(-) diff --git a/packages/mongodb-downloader/src/index.ts b/packages/mongodb-downloader/src/index.ts index 53007efb..6b73236b 100644 --- a/packages/mongodb-downloader/src/index.ts +++ b/packages/mongodb-downloader/src/index.ts @@ -26,7 +26,7 @@ export type MongoDBDownloaderOptions = { /** The directory to download the artifacts to. */ directory: string; /** The semantic version specifier for the target version. */ - version: string; + version?: string; /** Whether to use a lockfile for preventing concurrent downloads of the same version. */ useLockfile: boolean; /** The options to pass to the download URL lookup. */ @@ -36,7 +36,7 @@ export type MongoDBDownloaderOptions = { export class MongoDBDownloader { async downloadMongoDbWithVersionInfo({ downloadOptions = {}, - version, + version = '*', directory, useLockfile, }: MongoDBDownloaderOptions): Promise { @@ -199,13 +199,6 @@ export class MongoDBDownloader { }); } - public async downloadMongoDb( - ...args: Parameters - ): Promise { - return (await this.downloadMongoDbWithVersionInfo(...args)) - .downloadedBinDir; - } - private async getCurrentDownloadedFile({ bindir, artifactInfoFile, @@ -238,7 +231,7 @@ const downloader = new MongoDBDownloader(); /** Download mongod + mongos with version info and return version info and the path to a directory containing them. */ export async function downloadMongoDbWithVersionInfo({ downloadOptions = {}, - version, + version = '*', directory, useLockfile, }: MongoDBDownloaderOptions): Promise { @@ -250,8 +243,18 @@ export async function downloadMongoDbWithVersionInfo({ }); } /** Download mongod + mongos and return the path to a directory containing them. */ -export async function downloadMongoDb( - ...args: Parameters -): Promise { - return await downloader.downloadMongoDb(...args); +export async function downloadMongoDb({ + downloadOptions = {}, + version = '*', + directory, + useLockfile, +}: MongoDBDownloaderOptions): Promise { + return ( + await downloader.downloadMongoDbWithVersionInfo({ + downloadOptions, + version, + directory, + useLockfile, + }) + ).downloadedBinDir; } diff --git a/packages/mongodb-runner/src/mongocluster.ts b/packages/mongodb-runner/src/mongocluster.ts index 4d698087..23ae02b2 100644 --- a/packages/mongodb-runner/src/mongocluster.ts +++ b/packages/mongodb-runner/src/mongocluster.ts @@ -36,7 +36,12 @@ export class MongoCluster { targetVersionSemverSpecifier?: string | undefined, options?: DownloadOptions | undefined, ): Promise { - return downloadMongoDb(tmpdir, targetVersionSemverSpecifier, options); + return downloadMongoDb({ + directory: tmpdir, + version: targetVersionSemverSpecifier, + downloadOptions: options, + useLockfile: true, + }); } serialize(): unknown /* JSON-serializable */ { From 6e82751bb870ab1be7339bc5c09ef9b88085b6f2 Mon Sep 17 00:00:00 2001 From: gagik Date: Thu, 23 Oct 2025 10:58:44 +0200 Subject: [PATCH 11/12] chore: fixup naming logic --- packages/mongodb-downloader/src/index.spec.ts | 8 ++++---- packages/mongodb-downloader/src/index.ts | 1 + 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/packages/mongodb-downloader/src/index.spec.ts b/packages/mongodb-downloader/src/index.spec.ts index 62a58ee5..e256eed0 100644 --- a/packages/mongodb-downloader/src/index.spec.ts +++ b/packages/mongodb-downloader/src/index.spec.ts @@ -307,21 +307,21 @@ describe('MongoDBDownloader', function () { { version: '8.1.0-enterprise', enterprise: undefined, - expectedVersion: '8.1.0-enterprise', - expectedVersionName: '8.1.0', + expectedVersion: '8.1.0', + expectedVersionName: '8.1.0-enterprise', expectedEnterpriseFlag: true, }, { version: '8.1.0-enterprise', enterprise: false, - expectedVersion: '8.1.0-enterprise', + expectedVersion: '8.1.0', expectedVersionName: '8.1.0-enterprise', expectedEnterpriseFlag: true, }, { version: '8.1.0-enterprise', enterprise: true, - expectedVersion: '8.1.0-enterprise', + expectedVersion: '8.1.0', expectedVersionName: '8.1.0-enterprise', expectedEnterpriseFlag: true, }, diff --git a/packages/mongodb-downloader/src/index.ts b/packages/mongodb-downloader/src/index.ts index 6b73236b..d9be7719 100644 --- a/packages/mongodb-downloader/src/index.ts +++ b/packages/mongodb-downloader/src/index.ts @@ -50,6 +50,7 @@ export class MongoDBDownloader { if (/-enterprise$/.test(version)) { isEnterprise = true; + version = version.replace(/-enterprise$/, ''); versionName = versionName.replace(/-enterprise$/, ''); } From 10afb7c3f16e0474509b4b43d3b103ede4863814 Mon Sep 17 00:00:00 2001 From: gagik Date: Fri, 24 Oct 2025 17:09:10 +0200 Subject: [PATCH 12/12] chore: update readme --- packages/mongodb-downloader/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/mongodb-downloader/README.md b/packages/mongodb-downloader/README.md index d853d1e7..711c7af8 100644 --- a/packages/mongodb-downloader/README.md +++ b/packages/mongodb-downloader/README.md @@ -2,12 +2,12 @@ A simple library to download MongoDB binaries for different platforms and versions. -### Migrating from v0.4 to v0.5 +### Migrating from v0.5 to v0.6+ -In v0.5.x, the library introduced lockfiles to prevent parallel downloads of the same MongoDB binary. It also changed the arguments for the `downloadMongoDb` and `downloadMongoDbWithVersionInfo` functions, introducing a new `useLockfile` field and using a single options object instead of separate parameters for different options. It is recommended to enable lockfiles to prevent redundant downloads unless you have a specific reason not to. +In v0.6.x, the library introduced lockfiles to prevent parallel downloads of the same MongoDB binary. It also changed the arguments for the `downloadMongoDb` and `downloadMongoDbWithVersionInfo` functions, introducing a new `useLockfile` field and using a single options object instead of separate parameters for different options. It is recommended to enable lockfiles to prevent redundant downloads unless you have a specific reason not to. ```ts -// Before (v0.4.x) +// Before (v0.5.x) downloadMongoDb('/tmp/directory', '4.4.6', { platform: 'linux', arch: 'x64', @@ -17,7 +17,7 @@ downloadMongoDbWithVersionInfo('/tmp/directory', '4.4.6', { arch: 'x64', }); -// After (v0.5.x) +// After (v0.6.x) downloadMongoDb({ directory: '/tmp/directory', version: '4.4.6',