diff --git a/package-lock.json b/package-lock.json index eededf2e..71d48032 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10373,6 +10373,16 @@ "integrity": "sha512-ga8y9v9uyeiLdpKddhxYQkxNDrfvuPrlFb0N1qnZZByvcElJaXthF1UhvCh9TLWJBEHeNtdnbysW7Y6Uq8CVng==", "dev": true }, + "node_modules/@types/proper-lockfile": { + "version": "4.1.4", + "resolved": "https://registry.npmjs.org/@types/proper-lockfile/-/proper-lockfile-4.1.4.tgz", + "integrity": "sha512-uo2ABllncSqg9F1D4nugVl9v93RmjxF6LJzQLMLDdPaXCUIDPeOJ21Gbqi43xNKzBi/WQ0Q0dICqufzQbMjipQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/retry": "*" + } + }, "node_modules/@types/react": { "version": "17.0.75", "resolved": "https://registry.npmjs.org/@types/react/-/react-17.0.75.tgz", @@ -10426,6 +10436,13 @@ "@types/node": "*" } }, + "node_modules/@types/retry": { + "version": "0.12.5", + "resolved": "https://registry.npmjs.org/@types/retry/-/retry-0.12.5.tgz", + "integrity": "sha512-3xSjTp3v03X/lSQLkczaN9UIEwJMoMCA1+Nb5HfbJEQWogdeQIyVtTvxPXDQjZ5zws8rFQfVfRdz03ARihPJgw==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/s3rver": { "version": "3.7.0", "resolved": "https://registry.npmjs.org/@types/s3rver/-/s3rver-3.7.0.tgz", @@ -25374,6 +25391,7 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/promise-retry/-/promise-retry-2.0.1.tgz", "integrity": "sha512-y+WKFlBR8BGXnsNlIHFGPZmyDf3DFMoLhaflAnyZgV6rG6xu+JwesTo2Q9R6XwYmtmwAFCkAk3e35jEdoeh/3g==", + "license": "MIT", "dependencies": { "err-code": "^2.0.2", "retry": "^0.12.0" @@ -25425,6 +25443,17 @@ "node": ">= 8" } }, + "node_modules/proper-lockfile": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/proper-lockfile/-/proper-lockfile-4.1.2.tgz", + "integrity": "sha512-TjNPblN4BwAWMXU8s9AEz4JmQxnD1NNL7bNOY/AKUzyamc379FWASUhc/K1pL2noVb+XmZKLL68cjzLsiOAMaA==", + "license": "MIT", + "dependencies": { + "graceful-fs": "^4.2.4", + "retry": "^0.12.0", + "signal-exit": "^3.0.2" + } + }, "node_modules/protocols": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/protocols/-/protocols-2.0.1.tgz", @@ -30715,6 +30744,7 @@ "decompress": "^4.2.1", "mongodb-download-url": "^1.7.0", "node-fetch": "^2.7.0", + "proper-lockfile": "^4.1.2", "tar": "^6.1.15" }, "devDependencies": { @@ -30722,17 +30752,23 @@ "@mongodb-js/mocha-config-devtools": "^1.0.5", "@mongodb-js/prettier-config-devtools": "^1.0.2", "@mongodb-js/tsconfig-devtools": "^1.0.4", + "@types/chai": "^4.2.21", "@types/debug": "^4.1.8", "@types/decompress": "^4.2.4", "@types/mocha": "^9.1.1", "@types/node": "^22.15.30", + "@types/proper-lockfile": "^4.1.4", + "@types/sinon-chai": "^3.2.5", "@types/tar": "^6.1.5", + "chai": "^4.5.0", "depcheck": "^1.4.7", "eslint": "^7.25.0", "gen-esm-wrapper": "^1.1.3", "mocha": "^8.4.0", "nyc": "^15.1.0", "prettier": "^3.5.3", + "sinon": "^9.2.3", + "sinon-chai": "^3.7.0", "typescript": "^5.0.4" } }, @@ -38926,11 +38962,15 @@ "@mongodb-js/mocha-config-devtools": "^1.0.5", "@mongodb-js/prettier-config-devtools": "^1.0.2", "@mongodb-js/tsconfig-devtools": "^1.0.4", + "@types/chai": "^4.2.21", "@types/debug": "^4.1.8", "@types/decompress": "^4.2.4", "@types/mocha": "^9.1.1", "@types/node": "^22.15.30", + "@types/proper-lockfile": "^4.1.4", + "@types/sinon-chai": "^3.2.5", "@types/tar": "^6.1.5", + "chai": "^4.5.0", "debug": "^4.4.0", "decompress": "^4.2.1", "depcheck": "^1.4.7", @@ -38941,6 +38981,9 @@ "node-fetch": "^2.7.0", "nyc": "^15.1.0", "prettier": "^3.5.3", + "proper-lockfile": "^4.1.2", + "sinon": "^9.2.3", + "sinon-chai": "^3.7.0", "tar": "^6.1.15", "typescript": "^5.0.4" }, @@ -41735,6 +41778,15 @@ "integrity": "sha512-ga8y9v9uyeiLdpKddhxYQkxNDrfvuPrlFb0N1qnZZByvcElJaXthF1UhvCh9TLWJBEHeNtdnbysW7Y6Uq8CVng==", "dev": true }, + "@types/proper-lockfile": { + "version": "4.1.4", + "resolved": "https://registry.npmjs.org/@types/proper-lockfile/-/proper-lockfile-4.1.4.tgz", + "integrity": "sha512-uo2ABllncSqg9F1D4nugVl9v93RmjxF6LJzQLMLDdPaXCUIDPeOJ21Gbqi43xNKzBi/WQ0Q0dICqufzQbMjipQ==", + "dev": true, + "requires": { + "@types/retry": "*" + } + }, "@types/react": { "version": "17.0.75", "resolved": "https://registry.npmjs.org/@types/react/-/react-17.0.75.tgz", @@ -41787,6 +41839,12 @@ "@types/node": "*" } }, + "@types/retry": { + "version": "0.12.5", + "resolved": "https://registry.npmjs.org/@types/retry/-/retry-0.12.5.tgz", + "integrity": "sha512-3xSjTp3v03X/lSQLkczaN9UIEwJMoMCA1+Nb5HfbJEQWogdeQIyVtTvxPXDQjZ5zws8rFQfVfRdz03ARihPJgw==", + "dev": true + }, "@types/s3rver": { "version": "3.7.0", "resolved": "https://registry.npmjs.org/@types/s3rver/-/s3rver-3.7.0.tgz", @@ -53218,6 +53276,16 @@ "integrity": "sha512-vGrhOavPSTz4QVNuBNdcNXePNdNMaO1xj9yBeH1ScQPjk/rhg9sSlCXPhMkFuaNNW/syTvYqsnbIJxMBfRbbag==", "dev": true }, + "proper-lockfile": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/proper-lockfile/-/proper-lockfile-4.1.2.tgz", + "integrity": "sha512-TjNPblN4BwAWMXU8s9AEz4JmQxnD1NNL7bNOY/AKUzyamc379FWASUhc/K1pL2noVb+XmZKLL68cjzLsiOAMaA==", + "requires": { + "graceful-fs": "^4.2.4", + "retry": "^0.12.0", + "signal-exit": "^3.0.2" + } + }, "protocols": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/protocols/-/protocols-2.0.1.tgz", diff --git a/packages/mongodb-downloader/README.md b/packages/mongodb-downloader/README.md new file mode 100644 index 00000000..711c7af8 --- /dev/null +++ b/packages/mongodb-downloader/README.md @@ -0,0 +1,39 @@ +## mongodb-downloader + +A simple library to download MongoDB binaries for different platforms and versions. + +### Migrating from v0.5 to v0.6+ + +In v0.6.x, the library introduced lockfiles to prevent parallel downloads of the same MongoDB binary. It also changed the arguments for the `downloadMongoDb` and `downloadMongoDbWithVersionInfo` functions, introducing a new `useLockfile` field and using a single options object instead of separate parameters for different options. It is recommended to enable lockfiles to prevent redundant downloads unless you have a specific reason not to. + +```ts +// Before (v0.5.x) +downloadMongoDb('/tmp/directory', '4.4.6', { + platform: 'linux', + arch: 'x64', +}); + +downloadMongoDbWithVersionInfo('/tmp/directory', '4.4.6', { + arch: 'x64', +}); + +// After (v0.6.x) +downloadMongoDb({ + directory: '/tmp/directory', + version: '4.4.6', + useLockfile: true, // New, required field. + downloadOptions: { + platform: 'linux', + arch: 'x64', + }, +}); + +downloadMongoDbWithVersionInfo({ + directory: '/tmp/directory', + version: '4.4.6', + useLockfile: true, // New, required field. + downloadOptions: { + arch: 'x64', + }, +}); +``` diff --git a/packages/mongodb-downloader/package.json b/packages/mongodb-downloader/package.json index 8cd1740d..a75c1a96 100644 --- a/packages/mongodb-downloader/package.json +++ b/packages/mongodb-downloader/package.json @@ -56,24 +56,31 @@ "tar": "^6.1.15", "decompress": "^4.2.1", "mongodb-download-url": "^1.7.0", - "node-fetch": "^2.7.0" + "node-fetch": "^2.7.0", + "proper-lockfile": "^4.1.2" }, "devDependencies": { "@mongodb-js/eslint-config-devtools": "0.9.12", "@mongodb-js/mocha-config-devtools": "^1.0.5", "@mongodb-js/prettier-config-devtools": "^1.0.2", + "@types/chai": "^4.2.21", "@mongodb-js/tsconfig-devtools": "^1.0.4", "@types/debug": "^4.1.8", "@types/decompress": "^4.2.4", "@types/mocha": "^9.1.1", "@types/node": "^22.15.30", + "@types/proper-lockfile": "^4.1.4", + "@types/sinon-chai": "^3.2.5", "@types/tar": "^6.1.5", + "chai": "^4.5.0", "depcheck": "^1.4.7", "eslint": "^7.25.0", "gen-esm-wrapper": "^1.1.3", "mocha": "^8.4.0", "nyc": "^15.1.0", "prettier": "^3.5.3", + "sinon": "^9.2.3", + "sinon-chai": "^3.7.0", "typescript": "^5.0.4" } } diff --git a/packages/mongodb-downloader/src/index.spec.ts b/packages/mongodb-downloader/src/index.spec.ts index e69de29b..e256eed0 100644 --- a/packages/mongodb-downloader/src/index.spec.ts +++ b/packages/mongodb-downloader/src/index.spec.ts @@ -0,0 +1,392 @@ +import chai, { expect } from 'chai'; +import { promises as fs } from 'fs'; +import path from 'path'; +import os from 'os'; +import sinon from 'sinon'; +import sinonChai from 'sinon-chai'; +import { MongoDBDownloader } from '.'; + +chai.use(sinonChai); + +describe('MongoDBDownloader', function () { + this.timeout(60_000); + + let directory: string; + let testDownloader: MongoDBDownloader; + let downloadAndExtractStub: sinon.SinonStub; + let lookupDownloadUrlStub: sinon.SinonStub; + + beforeEach(async function () { + directory = path.join( + os.tmpdir(), + `download-integration-tests-${Date.now()}`, + ); + await fs.mkdir(directory, { recursive: true }); + + // Create a test instance of the downloader + testDownloader = new MongoDBDownloader(); + + // Mock the downloadAndExtract method to avoid actual downloads + // eslint-disable-next-line @typescript-eslint/no-explicit-any + downloadAndExtractStub = sinon + .stub(testDownloader as any, 'downloadAndExtract') + .callsFake(async (...args: any[]) => { + // Create the bindir and a fake mongod executable + const params = args[0] as { + bindir: string; + url: string; + downloadTarget: string; + isCryptLibrary: boolean; + }; + await fs.mkdir(params.bindir, { recursive: true }); + await fs.writeFile( + path.join(params.bindir, 'mongod'), + '#!/bin/bash\necho "This is a mock mongod"', + ); + }); + + // Mock the lookupDownloadUrl method to avoid network calls + lookupDownloadUrlStub = sinon + .stub(testDownloader as any, 'lookupDownloadUrl') + .resolves({ + version: '8.2.0', + url: 'https://example.com/mongodb-8.2.0.tgz', + name: 'mongodb-8.2.0', + }); + }); + const version = '8.2.0'; + + afterEach(async function () { + // Restore stubs + downloadAndExtractStub.restore(); + lookupDownloadUrlStub.restore(); + + try { + await fs.rm(directory, { recursive: true }); + } catch { + // Ignore cleanup errors + } + }); + + describe('without lockfile', function () { + it('should download multiple times in parallel', async function () { + const results = await Promise.all([ + testDownloader.downloadMongoDbWithVersionInfo({ + directory, + version, + useLockfile: false, + }), + testDownloader.downloadMongoDbWithVersionInfo({ + directory, + version, + useLockfile: false, + }), + testDownloader.downloadMongoDbWithVersionInfo({ + directory, + version, + useLockfile: false, + }), + ]); + expect(results[0].version).to.equal(version); + expect(results[0].downloadedBinDir).to.be.a('string'); + + expect(downloadAndExtractStub).to.have.callCount(3); + }); + + it('should skip download if already completed sequentially', async function () { + const result = await testDownloader.downloadMongoDbWithVersionInfo({ + directory, + version, + useLockfile: false, + }); + + const result2 = await testDownloader.downloadMongoDbWithVersionInfo({ + directory, + version, + useLockfile: false, + }); + expect(result2.version).to.equal(version); + expect(result2.downloadedBinDir).to.equal(result.downloadedBinDir); + + expect(downloadAndExtractStub).to.have.been.calledOnce; + }); + + it('should handle different versions independently', async function () { + const version2 = '8.1.0'; + + // Update stub to return different version info for the second call + lookupDownloadUrlStub.onSecondCall().resolves({ + version: '8.1.0', + url: 'https://example.com/mongodb-8.1.0.tgz', + name: 'mongodb-8.1.0', + }); + + // Download different versions + const [result1, result2] = await Promise.all([ + testDownloader.downloadMongoDbWithVersionInfo({ + directory, + version, + useLockfile: false, + }), + testDownloader.downloadMongoDbWithVersionInfo({ + directory, + version: version2, + useLockfile: false, + }), + ]); + + expect(result1.version).to.not.equal(result2.version); + expect(result1.downloadedBinDir).to.not.equal(result2.downloadedBinDir); + + // Verify both downloaded directories exist and contain mongod + expect(await fs.stat(result1.downloadedBinDir)).to.be.ok; + expect(await fs.stat(path.join(result1.downloadedBinDir, 'mongod'))).to.be + .ok; + expect(await fs.stat(result2.downloadedBinDir)).to.be.ok; + expect(await fs.stat(path.join(result2.downloadedBinDir, 'mongod'))).to.be + .ok; + + // Verify downloadAndExtract was called twice (once for each version) + expect(downloadAndExtractStub).to.have.been.calledTwice; + }); + }); + + describe('with lockfile', function () { + it('should prevent concurrent downloads of the same version', async function () { + const results = await Promise.all([ + testDownloader.downloadMongoDbWithVersionInfo({ + directory, + version, + useLockfile: true, + }), + testDownloader.downloadMongoDbWithVersionInfo({ + directory, + version, + useLockfile: true, + }), + testDownloader.downloadMongoDbWithVersionInfo({ + directory, + version, + useLockfile: true, + }), + ]); + + // All results should be identical + expect(results[0].version).to.equal(version); + expect(results[1].version).to.equal(version); + expect(results[2].version).to.equal(version); + + expect(results[0].downloadedBinDir).to.equal(results[1].downloadedBinDir); + expect(results[1].downloadedBinDir).to.equal(results[2].downloadedBinDir); + + // Verify the downloaded directory exists and contains mongod + expect(await fs.stat(results[0].downloadedBinDir)).to.be.ok; + expect(await fs.stat(path.join(results[0].downloadedBinDir, 'mongod'))).to + .be.ok; + + // Verify downloadAndExtract was called only once despite 3 concurrent requests + expect(downloadAndExtractStub).to.have.been.calledOnce; + }); + + it('should wait for existing download to complete', async function () { + // First, download MongoDB normally + const result = await testDownloader.downloadMongoDbWithVersionInfo({ + directory, + version, + useLockfile: true, + }); + + expect(result.version).to.equal(version); + expect(result.downloadedBinDir).to.be.a('string'); + + // Verify the downloaded directory exists and contains mongod + expect(await fs.stat(result.downloadedBinDir)).to.be.ok; + expect(await fs.stat(path.join(result.downloadedBinDir, 'mongod'))).to.be + .ok; + + // Verify downloadAndExtract was called once + expect(downloadAndExtractStub).to.have.been.calledOnce; + }); + + it('should skip download if already completed', async function () { + // First download + const result1 = await testDownloader.downloadMongoDbWithVersionInfo({ + directory, + version, + useLockfile: true, + }); + + // Second download should use cached result + const result2 = await testDownloader.downloadMongoDbWithVersionInfo({ + directory, + version, + useLockfile: true, + }); + + expect(result1.version).to.equal(version); + expect(result2.version).to.equal(version); + + // Verify the downloaded directory exists and contains mongod + expect(await fs.stat(result1.downloadedBinDir)).to.be.ok; + expect(await fs.stat(path.join(result1.downloadedBinDir, 'mongod'))).to.be + .ok; + + // Verify downloadAndExtract was called only once, not twice + expect(downloadAndExtractStub).to.have.been.calledOnce; + }); + + it('should handle different versions independently', async function () { + const version2 = '8.1.0'; + + // Update stub to return different version info for the second call + lookupDownloadUrlStub.onSecondCall().resolves({ + version: '8.1.0', + url: 'https://example.com/mongodb-8.1.0.tgz', + name: 'mongodb-8.1.0', + }); + + // Download different versions + const [result1, result2] = await Promise.all([ + testDownloader.downloadMongoDbWithVersionInfo({ + directory, + version, + useLockfile: true, + }), + testDownloader.downloadMongoDbWithVersionInfo({ + directory, + version: version2, + useLockfile: true, + }), + ]); + + expect(result1.version).to.not.equal(result2.version); + expect(result1.downloadedBinDir).to.not.equal(result2.downloadedBinDir); + + // Verify both downloaded directories exist and contain mongod + expect(await fs.stat(result1.downloadedBinDir)).to.be.ok; + expect(await fs.stat(path.join(result1.downloadedBinDir, 'mongod'))).to.be + .ok; + expect(await fs.stat(result2.downloadedBinDir)).to.be.ok; + expect(await fs.stat(path.join(result2.downloadedBinDir, 'mongod'))).to.be + .ok; + + // Verify downloadAndExtract was called twice (once for each version) + expect(downloadAndExtractStub).to.have.been.calledTwice; + }); + }); + + describe('version name', function () { + for (const { + version, + enterprise, + expectedVersion, + expectedVersionName, + expectedEnterpriseFlag, + } of [ + { + version: '8.1.0', + enterprise: undefined, + expectedVersion: '8.1.0', + expectedVersionName: '8.1.0-community', + expectedEnterpriseFlag: false, + }, + { + version: '8.1.0', + enterprise: false, + expectedVersion: '8.1.0', + expectedVersionName: '8.1.0-community', + expectedEnterpriseFlag: false, + }, + { + version: '8.1.0', + enterprise: true, + expectedVersion: '8.1.0', + expectedVersionName: '8.1.0-enterprise', + expectedEnterpriseFlag: true, + }, + { + version: '8.1.0-enterprise', + enterprise: undefined, + expectedVersion: '8.1.0', + expectedVersionName: '8.1.0-enterprise', + expectedEnterpriseFlag: true, + }, + { + version: '8.1.0-enterprise', + enterprise: false, + expectedVersion: '8.1.0', + expectedVersionName: '8.1.0-enterprise', + expectedEnterpriseFlag: true, + }, + { + version: '8.1.0-enterprise', + enterprise: true, + expectedVersion: '8.1.0', + expectedVersionName: '8.1.0-enterprise', + expectedEnterpriseFlag: true, + }, + { + version: 'latest-alpha', + enterprise: undefined, + expectedVersion: 'latest-alpha', + expectedVersionName: 'latest-alpha', + expectedEnterpriseFlag: false, + }, + { + version: 'latest-alpha', + enterprise: true, + expectedVersion: 'latest-alpha', + expectedVersionName: 'latest-alpha', + expectedEnterpriseFlag: true, + }, + { + version: '7.0.5', + enterprise: false, + expectedVersion: '7.0.5', + expectedVersionName: '7.0.5-community', + expectedEnterpriseFlag: false, + }, + { + version: '8.1.0-rc0', + enterprise: false, + expectedVersion: '8.1.0-rc0', + expectedVersionName: '8.1.0-rc0-community', + expectedEnterpriseFlag: false, + }, + ]) { + it(`should resolve correct version for ${version} with enterprise=${String(enterprise)}`, async function () { + lookupDownloadUrlStub.resetHistory(); + + const opts: { + directory: string; + version: string; + useLockfile: boolean; + downloadOptions?: { enterprise: boolean }; + } = { + directory, + version, + useLockfile: false, + }; + + if (enterprise !== undefined) { + opts.downloadOptions = { enterprise: enterprise }; + } + + const result = + await testDownloader.downloadMongoDbWithVersionInfo(opts); + + // Verify lookup call + expect(lookupDownloadUrlStub).to.have.been.calledOnce; + + const callArgs = lookupDownloadUrlStub.firstCall.args[0]; + expect(callArgs.targetVersion).to.equal(expectedVersion); + expect(callArgs.enterprise).to.equal(expectedEnterpriseFlag); + + // Verify path contains expected string + expect(result.downloadedBinDir).to.include( + expectedVersionName.replaceAll('.', ''), + ); + }); + } + }); +}); diff --git a/packages/mongodb-downloader/src/index.ts b/packages/mongodb-downloader/src/index.ts index b554ed14..d9be7719 100644 --- a/packages/mongodb-downloader/src/index.ts +++ b/packages/mongodb-downloader/src/index.ts @@ -12,6 +12,8 @@ import type { DownloadArtifactInfo, } from 'mongodb-download-url'; import createDebug from 'debug'; +import { withLock } from './with-lock'; + const debug = createDebug('mongodb-downloader'); export type { DownloadOptions }; @@ -20,144 +22,240 @@ export type DownloadResult = DownloadArtifactInfo & { downloadedBinDir: string; }; -// Download mongod + mongos and return the path to a directory containing them. -export async function downloadMongoDbWithVersionInfo( - tmpdir: string, - targetVersionSemverSpecifier = '*', - options: DownloadOptions = {}, -): Promise { - let wantsEnterprise = options.enterprise ?? false; - const isWindows = ['win32', 'windows'].includes( - options.platform ?? process.platform, - ); - async function lookupDownloadUrl(): Promise { - return await getDownloadURL({ - version: targetVersionSemverSpecifier, - enterprise: wantsEnterprise, - ...options, - }); - } +export type MongoDBDownloaderOptions = { + /** The directory to download the artifacts to. */ + directory: string; + /** The semantic version specifier for the target version. */ + version?: string; + /** Whether to use a lockfile for preventing concurrent downloads of the same version. */ + useLockfile: boolean; + /** The options to pass to the download URL lookup. */ + downloadOptions?: DownloadOptions; +}; - await fs.mkdir(tmpdir, { recursive: true }); - if (targetVersionSemverSpecifier === 'latest-alpha') { - return await doDownload( - tmpdir, - !!options.crypt_shared, - 'latest-alpha', - isWindows, - lookupDownloadUrl, +export class MongoDBDownloader { + async downloadMongoDbWithVersionInfo({ + downloadOptions = {}, + version = '*', + directory, + useLockfile, + }: MongoDBDownloaderOptions): Promise { + await fs.mkdir(directory, { recursive: true }); + const isWindows = ['win32', 'windows'].includes( + downloadOptions.platform ?? process.platform, ); - } + const isCryptLibrary = !!downloadOptions.crypt_shared; + let isEnterprise = downloadOptions.enterprise ?? false; + let versionName = version; - if (/-enterprise$/.test(targetVersionSemverSpecifier)) { - wantsEnterprise = true; - targetVersionSemverSpecifier = targetVersionSemverSpecifier.replace( - /-enterprise$/, - '', - ); - } + if (/-enterprise$/.test(version)) { + isEnterprise = true; + version = version.replace(/-enterprise$/, ''); + versionName = versionName.replace(/-enterprise$/, ''); + } - return await doDownload( - tmpdir, - !!options.crypt_shared, - targetVersionSemverSpecifier + - (wantsEnterprise ? '-enterprise' : '-community'), - isWindows, - () => lookupDownloadUrl(), - ); -} + if (versionName !== 'latest-alpha') { + versionName = versionName + (isEnterprise ? '-enterprise' : '-community'); + } -const downloadPromises: Record> = Object.create( - null, -); -async function doDownload( - tmpdir: string, - isCryptLibrary: boolean, - version: string, - isWindows: boolean, - lookupDownloadUrl: () => Promise, -): Promise { - const downloadTarget = path.resolve( - tmpdir, - `mongodb-${process.platform}-${process.env.DISTRO_ID || 'none'}-${ - process.arch - }-${version}`.replace(/[^a-zA-Z0-9_-]/g, ''), - ); - return (downloadPromises[downloadTarget] ??= (async () => { + const downloadTarget = path.resolve( + directory, + `mongodb-${process.platform}-${process.env.DISTRO_ID || 'none'}-${ + process.arch + }-${versionName}`.replace(/[^a-zA-Z0-9_-]/g, ''), + ); const bindir = path.resolve( downloadTarget, isCryptLibrary && !isWindows ? 'lib' : 'bin', ); - const artifactInfoFile = path.join(bindir, '.artifact_info'); + + return (async () => { + const artifactInfoFile = path.join(bindir, '.artifact_info'); + + // Check if already downloaded before acquiring lock + const currentDownloadedFile = await this.getCurrentDownloadedFile({ + bindir, + artifactInfoFile, + }); + if (currentDownloadedFile) { + debug(`Skipping download because ${downloadTarget} exists`); + return currentDownloadedFile; + } + + // Acquire the lock and perform download + return await (useLockfile ? withLock : withoutLock)( + downloadTarget, + async () => { + // Check again inside lock in case another process downloaded it + const downloadedFile = await this.getCurrentDownloadedFile({ + bindir, + artifactInfoFile, + }); + if (downloadedFile) { + debug( + `Skipping download because ${downloadTarget} exists after waiting on lock`, + ); + return downloadedFile; + } + + await fs.mkdir(downloadTarget, { recursive: true }); + const artifactInfo = await this.lookupDownloadUrl({ + targetVersion: version, + enterprise: isEnterprise, + options: downloadOptions, + }); + const { url } = artifactInfo; + debug(`Downloading ${url} into ${downloadTarget}...`); + + await this.downloadAndExtract({ + url, + downloadTarget, + isCryptLibrary, + bindir, + }); + await fs.writeFile(artifactInfoFile, JSON.stringify(artifactInfo)); + debug(`Download complete`, bindir); + return { ...artifactInfo, downloadedBinDir: bindir }; + }, + ); + })(); + } + + static HWM = 1024 * 1024; + + private async downloadAndExtract({ + withExtraStripDepth = 0, + downloadTarget, + isCryptLibrary, + bindir, + url, + }: { + withExtraStripDepth?: number; + downloadTarget: string; + isCryptLibrary: boolean; + bindir: string; + url: string; + }): Promise { + const response = await fetch(url, { + highWaterMark: MongoDBDownloader.HWM, + } as Parameters[1]); + if (/\.tgz$|\.tar(\.[^.]+)?$/.exec(url)) { + // the server's tarballs can contain hard links, which the (unmaintained?) + // `download` package is unable to handle (https://github.com/kevva/decompress/issues/93) + await promisify(pipeline)( + response.body, + tar.x({ cwd: downloadTarget, strip: isCryptLibrary ? 0 : 1 }), + ); + } else { + const filename = path.join( + downloadTarget, + path.basename(new URL(url).pathname), + ); + await promisify(pipeline)( + response.body, + createWriteStream(filename, { highWaterMark: MongoDBDownloader.HWM }), + ); + debug(`Written file ${url} to ${filename}, extracting...`); + await decompress(filename, downloadTarget, { + strip: isCryptLibrary ? 0 : 1, + filter: (file) => path.extname(file.path) !== '.pdb', // Windows .pdb files are huge and useless + }); + } + + try { + await fs.stat(bindir); // Make sure it exists. + } catch (err) { + if (withExtraStripDepth === 0 && url.includes('macos')) { + // The server team changed how macos release artifacts are packed + // and added a `./` prefix to paths in the tarball, + // which seems like it shouldn't change anything but does + // in fact require an increased path strip depth. + // eslint-disable-next-line no-console + console.info('Retry due to miscalculated --strip-components depth'); + return await this.downloadAndExtract({ + withExtraStripDepth: 1, + url, + downloadTarget, + isCryptLibrary, + bindir, + }); + } + throw err; + } + } + + private async lookupDownloadUrl({ + targetVersion, + enterprise, + options, + }: { + targetVersion: string; + enterprise: boolean; + options: DownloadOptions; + }): Promise { + return await getDownloadURL({ + version: targetVersion, + enterprise, + ...options, + }); + } + + private async getCurrentDownloadedFile({ + bindir, + artifactInfoFile, + }: { + bindir: string; + artifactInfoFile: string; + }): Promise { try { await fs.stat(artifactInfoFile); - debug(`Skipping download because ${downloadTarget} exists`); return { ...JSON.parse(await fs.readFile(artifactInfoFile, 'utf8')), downloadedBinDir: bindir, }; } catch { - /* ignore */ + /* ignore - file doesn't exist, proceed with download */ } + } +} - await fs.mkdir(downloadTarget, { recursive: true }); - const artifactInfo = await lookupDownloadUrl(); - const { url } = artifactInfo; - debug(`Downloading ${url} into ${downloadTarget}...`); - - // Using a large highWaterMark setting noticeably speeds up Windows downloads - const HWM = 1024 * 1024; - async function downloadAndExtract(withExtraStripDepth = 0): Promise { - const response = await fetch(url, { - highWaterMark: HWM, - } as any); - if (/\.tgz$|\.tar(\.[^.]+)?$/.exec(url)) { - // the server's tarballs can contain hard links, which the (unmaintained?) - // `download` package is unable to handle (https://github.com/kevva/decompress/issues/93) - await promisify(pipeline)( - response.body, - tar.x({ cwd: downloadTarget, strip: isCryptLibrary ? 0 : 1 }), - ); - } else { - const filename = path.join( - downloadTarget, - path.basename(new URL(url).pathname), - ); - await promisify(pipeline)( - response.body, - createWriteStream(filename, { highWaterMark: HWM }), - ); - debug(`Written file ${url} to ${filename}, extracting...`); - await decompress(filename, downloadTarget, { - strip: isCryptLibrary ? 0 : 1, - filter: (file) => path.extname(file.path) !== '.pdb', // Windows .pdb files are huge and useless - }); - } +/** Runs the callback without a lock, using same interface as `withLock` */ +async function withoutLock( + bindir: string, + callback: () => Promise, +): Promise { + return await callback(); +} - try { - await fs.stat(bindir); // Make sure it exists. - } catch (err) { - if (withExtraStripDepth === 0 && url.includes('macos')) { - // The server team changed how macos release artifacts are packed - // and added a `./` prefix to paths in the tarball, - // which seems like it shouldn't change anything but does - // in fact require an increased path strip depth. - console.info('Retry due to miscalculated --strip-components depth'); - return await downloadAndExtract(1); - } - throw err; - } - } +const downloader = new MongoDBDownloader(); - await downloadAndExtract(); - await fs.writeFile(artifactInfoFile, JSON.stringify(artifactInfo)); - debug(`Download complete`, bindir); - return { ...artifactInfo, downloadedBinDir: bindir }; - })()); +/** Download mongod + mongos with version info and return version info and the path to a directory containing them. */ +export async function downloadMongoDbWithVersionInfo({ + downloadOptions = {}, + version = '*', + directory, + useLockfile, +}: MongoDBDownloaderOptions): Promise { + return await downloader.downloadMongoDbWithVersionInfo({ + downloadOptions, + version, + directory, + useLockfile, + }); } - -export async function downloadMongoDb( - ...args: Parameters -): Promise { - return (await downloadMongoDbWithVersionInfo(...args)).downloadedBinDir; +/** Download mongod + mongos and return the path to a directory containing them. */ +export async function downloadMongoDb({ + downloadOptions = {}, + version = '*', + directory, + useLockfile, +}: MongoDBDownloaderOptions): Promise { + return ( + await downloader.downloadMongoDbWithVersionInfo({ + downloadOptions, + version, + directory, + useLockfile, + }) + ).downloadedBinDir; } diff --git a/packages/mongodb-downloader/src/with-lock.ts b/packages/mongodb-downloader/src/with-lock.ts new file mode 100644 index 00000000..0e27e94e --- /dev/null +++ b/packages/mongodb-downloader/src/with-lock.ts @@ -0,0 +1,61 @@ +import lockfile from 'proper-lockfile'; +import fs from 'fs/promises'; + +/** + * Acquire an advisory lock for the given path and hold it for the duration of the callback. + * + * The lock will be released automatically when the callback resolves or rejects. + * Concurrent calls to withLock() for the same path will wait until the lock is released. + */ +export async function withLock( + originalFilePath: string, + cb: (signal: AbortSignal) => Promise, +): Promise { + const controller = new AbortController(); + let release: (() => Promise) | undefined; + + const lockFile = `${originalFilePath}/.mongodb-downloader-lock`; + + // Create the directory if it doesn't exist + await fs.mkdir(originalFilePath, { recursive: true }); + + // Create a dummy lock file to prevent concurrent downloads of the same version + await fs.writeFile(lockFile, ''); + + try { + release = await lockfile.lock(lockFile, { + retries: { + retries: 100, + minTimeout: 100, + maxTimeout: 5000, + factor: 2, + }, + stale: 100_000, + onCompromised: () => { + controller.abort(); + }, + }); + + return await new Promise((resolve, reject) => { + controller.signal.addEventListener('abort', () => { + reject(new Error('Aborted by abort signal')); + }); + + void (async () => { + try { + resolve(await cb(controller.signal)); + } catch (err) { + reject(err); + } + })(); + }); + } finally { + if (release) { + try { + await release(); + } catch (err) { + // Ignore errors during release + } + } + } +} diff --git a/packages/mongodb-runner/src/mongocluster.ts b/packages/mongodb-runner/src/mongocluster.ts index 4d698087..23ae02b2 100644 --- a/packages/mongodb-runner/src/mongocluster.ts +++ b/packages/mongodb-runner/src/mongocluster.ts @@ -36,7 +36,12 @@ export class MongoCluster { targetVersionSemverSpecifier?: string | undefined, options?: DownloadOptions | undefined, ): Promise { - return downloadMongoDb(tmpdir, targetVersionSemverSpecifier, options); + return downloadMongoDb({ + directory: tmpdir, + version: targetVersionSemverSpecifier, + downloadOptions: options, + useLockfile: true, + }); } serialize(): unknown /* JSON-serializable */ {