diff --git a/packages/build/src/barque.spec.ts b/packages/build/src/barque.spec.ts index 796ee7dc99..e0be18248d 100644 --- a/packages/build/src/barque.spec.ts +++ b/packages/build/src/barque.spec.ts @@ -49,17 +49,38 @@ describe('Barque', () => { describe('releaseToBarque', () => { context('platform is linux', () => { - it('execCurator function succeeds', async() => { - barque.execCurator = sinon.stub().returns(Promise.resolve(true)); - barque.createCuratorDir = sinon.stub().returns(Promise.resolve('./')); - barque.extractLatestCurator = sinon.stub().returns(Promise.resolve(true)); + context('execCurator function succeeds', () => { + [ + { + variant: BuildVariant.Debian, + url: 'https://s3.amazonaws.com/mciuploads/mongosh/5ed7ee5d8683818eb28d9d3b5c65837cde4a08f5/mongosh_0.1.0_amd64.deb', + publishedUrls: [ + `${Barque.PPA_REPO_BASE_URL}/apt/debian/dists/buster/mongodb-org/4.4/main/binary-amd64/mongosh_0.1.0_amd64.deb`, + `${Barque.PPA_REPO_BASE_URL}/apt/ubuntu/dists/bionic/mongodb-org/4.4/multiverse/binary-amd64/mongosh_0.1.0_amd64.deb`, + `${Barque.PPA_REPO_BASE_URL}/apt/ubuntu/dists/focal/mongodb-org/4.4/multiverse/binary-amd64/mongosh_0.1.0_amd64.deb`, + ] + }, + { + variant: BuildVariant.Redhat, + url: 'https://s3.amazonaws.com/mciuploads/mongosh/5ed7ee5d8683818eb28d9d3b5c65837cde4a08f5/mongosh-0.1.0-x86_64.rpm', + publishedUrls: [ + `${Barque.PPA_REPO_BASE_URL}/yum/redhat/8/mongodb-org/4.4/x86_64/RPMS/mongosh-0.1.0-x86_64.rpm`, + ] + } + ].forEach(({ variant, url, publishedUrls }) => { + it(`publishes ${variant} packages`, async() => { + barque.execCurator = sinon.stub().resolves(true); + barque.createCuratorDir = sinon.stub().resolves('./'); + barque.extractLatestCurator = sinon.stub().resolves(true); - const debUrl = 'https://s3.amazonaws.com/mciuploads/mongosh/5ed7ee5d8683818eb28d9d3b5c65837cde4a08f5/mongosh_0.1.0_amd64.deb'; + const releasedUrls = await barque.releaseToBarque(variant, url); - await barque.releaseToBarque(BuildVariant.Debian, debUrl); - expect(barque.createCuratorDir).to.have.been.called; - expect(barque.extractLatestCurator).to.have.been.called; - expect(barque.execCurator).to.have.been.called; + expect(releasedUrls).to.deep.equal(publishedUrls); + expect(barque.createCuratorDir).to.have.been.called; + expect(barque.extractLatestCurator).to.have.been.called; + expect(barque.execCurator).to.have.been.called; + }); + }); }); it('execCurator function fails', async() => { @@ -84,18 +105,13 @@ describe('Barque', () => { it('platform is not linux', async() => { config.platform = 'macos'; - barque = new Barque(config); - - barque.execCurator = sinon.stub().returns(Promise.resolve(true)); - barque.createCuratorDir = sinon.stub().returns(Promise.resolve('./')); - barque.extractLatestCurator = sinon.stub().returns(Promise.resolve(true)); - - const debUrl = 'https://s3.amazonaws.com/mciuploads/mongosh/5ed7ee5d8683818eb28d9d3b5c65837cde4a08f5/mongosh_0.1.0_linux.deb'; - - await barque.releaseToBarque(BuildVariant.Debian, debUrl); - expect(barque.createCuratorDir).to.not.have.been.called; - expect(barque.extractLatestCurator).to.not.have.been.called; - expect(barque.execCurator).to.not.have.been.called; + try { + barque = new Barque(config); + } catch (e) { + expect(e.message).to.contain('only supported on linux'); + return; + } + expect.fail('Expected error'); }); }); @@ -165,6 +181,60 @@ describe('Barque', () => { }); }); + describe('waitUntilPackagesAreAvailable', () => { + beforeEach(() => { + nock.cleanAll(); + }); + + context('with packages published one after the other', () => { + let nockRepo: nock.Scope; + + beforeEach(() => { + nockRepo = nock(Barque.PPA_REPO_BASE_URL); + + nockRepo.head('/apt/dist/package1.deb').reply(200); + + nockRepo.head('/apt/dist/package2.deb').twice().reply(404); + nockRepo.head('/apt/dist/package2.deb').reply(200); + + nockRepo.head('/apt/dist/package3.deb').reply(404); + nockRepo.head('/apt/dist/package3.deb').reply(200); + }); + + it('waits until all packages are available', async() => { + await barque.waitUntilPackagesAreAvailable([ + `${Barque.PPA_REPO_BASE_URL}/apt/dist/package1.deb`, + `${Barque.PPA_REPO_BASE_URL}/apt/dist/package2.deb`, + `${Barque.PPA_REPO_BASE_URL}/apt/dist/package3.deb` + ], 300, 1); + + expect(nock.isDone()).to.be.true; + }); + }); + + context('with really slow packages', () => { + let nockRepo: nock.Scope; + + beforeEach(() => { + nockRepo = nock(Barque.PPA_REPO_BASE_URL); + nockRepo.head('/apt/dist/package1.deb').reply(200); + nockRepo.head('/apt/dist/package2.deb').reply(404).persist(); + }); + + it('fails when the timeout is hit', async() => { + try { + await barque.waitUntilPackagesAreAvailable([ + `${Barque.PPA_REPO_BASE_URL}/apt/dist/package1.deb`, + `${Barque.PPA_REPO_BASE_URL}/apt/dist/package2.deb`, + ], 5, 1); + } catch (e) { + expect(e.message).to.contain('the following packages are still not available'); + expect(e.message).to.contain('package2.deb'); + } + }); + }); + }); + describe('LATEST_CURATOR', () => { it('can be downloaded', async() => { const response = await fetch(LATEST_CURATOR, { diff --git a/packages/build/src/barque.ts b/packages/build/src/barque.ts index d22cb7e0af..d343d9668c 100644 --- a/packages/build/src/barque.ts +++ b/packages/build/src/barque.ts @@ -6,7 +6,7 @@ import path from 'path'; import stream from 'stream'; import tar from 'tar-fs'; import tmp from 'tmp-promise'; -import util from 'util'; +import util, { promisify } from 'util'; import { BuildVariant, Config, Platform } from './config'; const pipeline = util.promisify(stream.pipeline); @@ -42,11 +42,17 @@ enum Arch { } export class Barque { + public static readonly PPA_REPO_BASE_URL = 'https://repo.mongodb.org' as const; + private config: Config; private mongodbEdition: string; private mongodbVersion: string; constructor(config: Config) { + if (config.platform !== Platform.Linux) { + throw new Error('Barque publishing is only supported on linux platforms'); + } + this.config = config; // hard code mongodb edition to 'org' for now this.mongodbEdition = 'org'; @@ -56,29 +62,31 @@ export class Barque { } /** - * Upload current package to barque, MongoDB's PPA for linux distros. + * Upload a distributable package to barque, MongoDB's PPA for linux distros. + * + * Note that this method returns the URLs where the packages _will_ be available. + * This method does not wait for the packages to really be available. + * Use `waitUntilPackagesAreAvailable` for this purpose. * - * @param {string} tarballURL- The uploaded to Evergreen tarball URL. - * @param {Config} config - Config object. + * @param buildVariant - The distributable package build variant to publish. + * @param packageUrl - The Evergreen URL of the distributable package. * - * @returns {Promise} The promise. + * @returns The URLs where the packages will be available. */ - async releaseToBarque(buildVariant: BuildVariant, tarballURL: string): Promise { - if (this.config.platform !== Platform.Linux) { - return; - } - + async releaseToBarque(buildVariant: BuildVariant, packageUrl: string): Promise { const repoConfig = path.join(this.config.rootDir, 'config', 'repo-config.yml'); const curatorDirPath = await this.createCuratorDir(); await this.extractLatestCurator(curatorDirPath); const targetDistros = this.getTargetDistros(buildVariant); const targetArchitecture = this.getTargetArchitecture(buildVariant); + + const publishedPackageUrls: string[] = []; for (const distro of targetDistros) { try { await this.execCurator( curatorDirPath, - tarballURL, + packageUrl, repoConfig, distro, targetArchitecture @@ -87,12 +95,15 @@ export class Barque { console.error('Curator failed', error); throw new Error(`Curator is unable to upload to barque ${error}`); } + + publishedPackageUrls.push(this.computePublishedPackageUrl(distro, targetArchitecture, packageUrl)); } + return publishedPackageUrls; } async execCurator( curatorDirPath: string, - tarballURL: string, + packageUrl: string, repoConfig: string, distro: Distro, architecture: Arch @@ -107,7 +118,7 @@ export class Barque { '--arch', architecture, '--edition', this.mongodbEdition, '--version', this.mongodbVersion, - '--packages', tarballURL + '--packages', packageUrl ], { // curator looks for these options in env env: { @@ -147,6 +158,61 @@ export class Barque { } } + computePublishedPackageUrl(distro: Distro, targetArchitecture: Arch, packageUrl: string): string { + const packageFileName = packageUrl.split('/').slice(-1); + const packageFolderVersion = this.mongodbVersion.split('.').slice(0, 2).join('.'); + switch (distro) { + case Distro.Debian10: + return `${Barque.PPA_REPO_BASE_URL}/apt/debian/dists/buster/mongodb-org/${packageFolderVersion}/main/binary-${targetArchitecture}/${packageFileName}`; + case Distro.Ubuntu1804: + return `${Barque.PPA_REPO_BASE_URL}/apt/ubuntu/dists/bionic/mongodb-org/${packageFolderVersion}/multiverse/binary-${targetArchitecture}/${packageFileName}`; + case Distro.Ubuntu2004: + return `${Barque.PPA_REPO_BASE_URL}/apt/ubuntu/dists/focal/mongodb-org/${packageFolderVersion}/multiverse/binary-${targetArchitecture}/${packageFileName}`; + case Distro.Redhat80: + return `${Barque.PPA_REPO_BASE_URL}/yum/redhat/8/mongodb-org/${packageFolderVersion}/${targetArchitecture}/RPMS/${packageFileName}`; + default: + throw new Error(`Unsupported distro: ${distro}`); + } + } + + /** + * Waits until the given packages are available under the specified URLs or throws an error if there + * are still remaining packages after the timeout. + * + * Note that the method will try all URLs at least once after an initial delay of `sleepTimeSeconds`. + */ + async waitUntilPackagesAreAvailable(publishedPackageUrls: string[], timeoutSeconds: number, sleepTimeSeconds = 10): Promise { + let remainingPackages = [...publishedPackageUrls]; + const sleep = promisify(setTimeout); + + const startMs = new Date().getTime(); + const failOnTimeout = () => { + if (new Date().getTime() - startMs > timeoutSeconds * 1000) { + throw new Error(`Barque timed out - the following packages are still not available: ${remainingPackages.join(', ')}`); + } + }; + + while (remainingPackages.length) { + console.info(`Waiting for availability of:\n - ${remainingPackages.join('\n - ')}`); + await sleep(sleepTimeSeconds * 1000); + + const promises = remainingPackages.map(async url => await fetch(url, { + method: 'HEAD' + })); + const responses = await Promise.all(promises); + + const newRemainingPackages: string[] = []; + for (let i = 0; i < remainingPackages.length; i++) { + if (responses[i].status !== 200) { + newRemainingPackages.push(remainingPackages[i]); + } + } + remainingPackages = newRemainingPackages; + + failOnTimeout(); + } + } + /** * Create a staging dir in /tmp to download the latest version of curator. * diff --git a/packages/build/src/run-publish.spec.ts b/packages/build/src/run-publish.spec.ts index 0ff6b4281f..e3baeb5500 100644 --- a/packages/build/src/run-publish.spec.ts +++ b/packages/build/src/run-publish.spec.ts @@ -81,7 +81,8 @@ describe('publish', () => { githubRepo = createStubRepo(); mongoHomebrewRepo = createStubRepo(); barque = createStubBarque({ - releaseToBarque: sinon.stub().resolves(true) + releaseToBarque: sinon.stub().resolves(['package-url']), + waitUntilPackagesAreAvailable: sinon.stub().resolves() }); }); @@ -188,6 +189,7 @@ describe('publish', () => { BuildVariant.Debian, 'https://s3.amazonaws.com/mciuploads/project/v0.7.0-draft.42/mongosh_0.7.0_amd64.deb' ); + expect(barque.waitUntilPackagesAreAvailable).to.have.been.called; }); it('updates the download center config', async() => { diff --git a/packages/build/src/run-publish.ts b/packages/build/src/run-publish.ts index 703d75253c..85531241db 100644 --- a/packages/build/src/run-publish.ts +++ b/packages/build/src/run-publish.ts @@ -92,11 +92,17 @@ async function publishArtifactsToBarque( BuildVariant.Debian, BuildVariant.Redhat ]; + + const publishedPackages: string[] = []; for await (const variant of variantsForBarque) { const tarballName = getTarballFile(variant, releaseVersion, packageName); const tarballUrl = getEvergreenArtifactUrl(project, mostRecentDraftTag, tarballName.path); console.info(`mongosh: Publishing ${variant} artifact to barque ${tarballUrl}`); - await barque.releaseToBarque(variant, tarballUrl); + const packageUrls = await barque.releaseToBarque(variant, tarballUrl); + publishedPackages.push(...packageUrls); } + + await barque.waitUntilPackagesAreAvailable(publishedPackages, 300); + console.info('mongosh: Submitting to barque complete'); }