chore(build): wait until packages are really available in PPAs MONGOSH-619

packages/build/src/barque.spec.ts

@@ -49,17 +49,38 @@ describe('Barque', () => {
 
   describe('releaseToBarque', () => {
     context('platform is linux', () => {
-      it('execCurator function succeeds', async() => {
-        barque.execCurator = sinon.stub().returns(Promise.resolve(true));
-        barque.createCuratorDir = sinon.stub().returns(Promise.resolve('./'));
-        barque.extractLatestCurator = sinon.stub().returns(Promise.resolve(true));
+      context('execCurator function succeeds', () => {
+        [
+          {
+            variant: BuildVariant.Debian,
+            url: 'https://s3.amazonaws.com/mciuploads/mongosh/5ed7ee5d8683818eb28d9d3b5c65837cde4a08f5/mongosh_0.1.0_amd64.deb',
+            publishedUrls: [
+              `${Barque.PPA_REPO_BASE_URL}/apt/debian/dists/buster/mongodb-org/4.4/main/binary-amd64/mongosh_0.1.0_amd64.deb`,
+              `${Barque.PPA_REPO_BASE_URL}/apt/ubuntu/dists/bionic/mongodb-org/4.4/multiverse/binary-amd64/mongosh_0.1.0_amd64.deb`,
+              `${Barque.PPA_REPO_BASE_URL}/apt/ubuntu/dists/focal/mongodb-org/4.4/multiverse/binary-amd64/mongosh_0.1.0_amd64.deb`,
+            ]
+          },
+          {
+            variant: BuildVariant.Redhat,
+            url: 'https://s3.amazonaws.com/mciuploads/mongosh/5ed7ee5d8683818eb28d9d3b5c65837cde4a08f5/mongosh-0.1.0-x86_64.rpm',
+            publishedUrls: [
+              `${Barque.PPA_REPO_BASE_URL}/yum/redhat/8/mongodb-org/4.4/x86_64/RPMS/mongosh-0.1.0-x86_64.rpm`,
+            ]
+          }
+        ].forEach(({ variant, url, publishedUrls }) => {
+          it(`publishes ${variant} packages`, async() => {
+            barque.execCurator = sinon.stub().resolves(true);
+            barque.createCuratorDir = sinon.stub().resolves('./');
+            barque.extractLatestCurator = sinon.stub().resolves(true);
 
-        const debUrl = 'https://s3.amazonaws.com/mciuploads/mongosh/5ed7ee5d8683818eb28d9d3b5c65837cde4a08f5/mongosh_0.1.0_amd64.deb';
+            const releasedUrls = await barque.releaseToBarque(variant, url);
 
-        await barque.releaseToBarque(BuildVariant.Debian, debUrl);
-        expect(barque.createCuratorDir).to.have.been.called;
-        expect(barque.extractLatestCurator).to.have.been.called;
-        expect(barque.execCurator).to.have.been.called;
+            expect(releasedUrls).to.deep.equal(publishedUrls);
+            expect(barque.createCuratorDir).to.have.been.called;
+            expect(barque.extractLatestCurator).to.have.been.called;
+            expect(barque.execCurator).to.have.been.called;
+          });
+        });
       });
 
       it('execCurator function fails', async() => {
@@ -84,18 +105,13 @@ describe('Barque', () => {
 
     it('platform is not linux', async() => {
       config.platform = 'macos';
-      barque = new Barque(config);
-
-      barque.execCurator = sinon.stub().returns(Promise.resolve(true));
-      barque.createCuratorDir = sinon.stub().returns(Promise.resolve('./'));
-      barque.extractLatestCurator = sinon.stub().returns(Promise.resolve(true));
-
-      const debUrl = 'https://s3.amazonaws.com/mciuploads/mongosh/5ed7ee5d8683818eb28d9d3b5c65837cde4a08f5/mongosh_0.1.0_linux.deb';
-
-      await barque.releaseToBarque(BuildVariant.Debian, debUrl);
-      expect(barque.createCuratorDir).to.not.have.been.called;
-      expect(barque.extractLatestCurator).to.not.have.been.called;
-      expect(barque.execCurator).to.not.have.been.called;
+      try {
+        barque = new Barque(config);
+      } catch (e) {
+        expect(e.message).to.contain('only supported on linux');
+        return;
+      }
+      expect.fail('Expected error');
     });
   });
 
@@ -165,6 +181,60 @@ describe('Barque', () => {
     });
   });
 
+  describe('waitUntilPackagesAreAvailable', () => {
+    beforeEach(() => {
+      nock.cleanAll();
+    });
+
+    context('with packages published one after the other', () => {
+      let nockRepo: nock.Scope;
+
+      beforeEach(() => {
+        nockRepo = nock(Barque.PPA_REPO_BASE_URL);
+
+        nockRepo.head('/apt/dist/package1.deb').reply(200);
+
+        nockRepo.head('/apt/dist/package2.deb').twice().reply(404);
+        nockRepo.head('/apt/dist/package2.deb').reply(200);
+
+        nockRepo.head('/apt/dist/package3.deb').reply(404);
+        nockRepo.head('/apt/dist/package3.deb').reply(200);
+      });
+
+      it('waits until all packages are available', async() => {
+        await barque.waitUntilPackagesAreAvailable([
+          `${Barque.PPA_REPO_BASE_URL}/apt/dist/package1.deb`,
+          `${Barque.PPA_REPO_BASE_URL}/apt/dist/package2.deb`,
+          `${Barque.PPA_REPO_BASE_URL}/apt/dist/package3.deb`
+        ], 300, 1);
+
+        expect(nock.isDone()).to.be.true;
+      });
+    });
+
+    context('with really slow packages', () => {
+      let nockRepo: nock.Scope;
+
+      beforeEach(() => {
+        nockRepo = nock(Barque.PPA_REPO_BASE_URL);
+        nockRepo.head('/apt/dist/package1.deb').reply(200);
+        nockRepo.head('/apt/dist/package2.deb').reply(404).persist();
+      });
+
+      it('fails when the timeout is hit', async() => {
+        try {
+          await barque.waitUntilPackagesAreAvailable([
+            `${Barque.PPA_REPO_BASE_URL}/apt/dist/package1.deb`,
+            `${Barque.PPA_REPO_BASE_URL}/apt/dist/package2.deb`,
+          ], 5, 1);
+        } catch (e) {
+          expect(e.message).to.contain('the following packages are still not available');
+          expect(e.message).to.contain('package2.deb');
+        }
+      });
+    });
+  });
+
   describe('LATEST_CURATOR', () => {
     it('can be downloaded', async() => {
       const response = await fetch(LATEST_CURATOR, {

packages/build/src/barque.ts

@@ -6,7 +6,7 @@ import path from 'path';
 import stream from 'stream';
 import tar from 'tar-fs';
 import tmp from 'tmp-promise';
-import util from 'util';
+import util, { promisify } from 'util';
 import { BuildVariant, Config, Platform } from './config';
 
 const pipeline = util.promisify(stream.pipeline);
@@ -42,11 +42,17 @@ enum Arch {
 }
 
 export class Barque {
+  public static readonly PPA_REPO_BASE_URL = 'https://repo.mongodb.org' as const;
+
   private config: Config;
   private mongodbEdition: string;
   private mongodbVersion: string;
 
   constructor(config: Config) {
+    if (config.platform !== Platform.Linux) {
+      throw new Error('Barque publishing is only supported on linux platforms');
+    }
+
     this.config = config;
     // hard code mongodb edition to 'org' for now
     this.mongodbEdition = 'org';
@@ -56,29 +62,31 @@ export class Barque {
   }
 
   /**
-   * Upload current package to barque, MongoDB's PPA for linux distros.
+   * Upload a distributable package to barque, MongoDB's PPA for linux distros.
+   *
+   * Note that this method returns the URLs where the packages _will_ be available.
+   * This method does not wait for the packages to really be available.
+   * Use `waitUntilPackagesAreAvailable` for this purpose.
    *
-   * @param {string} tarballURL- The uploaded to Evergreen tarball URL.
-   * @param {Config} config - Config object.
+   * @param buildVariant - The distributable package build variant to publish.
+   * @param packageUrl - The Evergreen URL of the distributable package.
    *
-   * @returns {Promise} The promise.
+   * @returns The URLs where the packages will be available.
    */
-  async releaseToBarque(buildVariant: BuildVariant, tarballURL: string): Promise<any> {
-    if (this.config.platform !== Platform.Linux) {
-      return;
-    }
-
+  async releaseToBarque(buildVariant: BuildVariant, packageUrl: string): Promise<string[]> {
     const repoConfig = path.join(this.config.rootDir, 'config', 'repo-config.yml');
     const curatorDirPath = await this.createCuratorDir();
     await this.extractLatestCurator(curatorDirPath);
 
     const targetDistros = this.getTargetDistros(buildVariant);
     const targetArchitecture = this.getTargetArchitecture(buildVariant);
+
+    const publishedPackageUrls: string[] = [];
     for (const distro of targetDistros) {
       try {
         await this.execCurator(
           curatorDirPath,
-          tarballURL,
+          packageUrl,
           repoConfig,
           distro,
           targetArchitecture
@@ -87,12 +95,15 @@ export class Barque {
         console.error('Curator failed', error);
         throw new Error(`Curator is unable to upload to barque ${error}`);
       }
+
+      publishedPackageUrls.push(this.computePublishedPackageUrl(distro, targetArchitecture, packageUrl));
     }
+    return publishedPackageUrls;
   }
 
   async execCurator(
     curatorDirPath: string,
-    tarballURL: string,
+    packageUrl: string,
     repoConfig: string,
     distro: Distro,
     architecture: Arch
@@ -107,7 +118,7 @@ export class Barque {
         '--arch', architecture,
         '--edition', this.mongodbEdition,
         '--version', this.mongodbVersion,
-        '--packages', tarballURL
+        '--packages', packageUrl
       ], {
         // curator looks for these options in env
         env: {
@@ -147,6 +158,61 @@ export class Barque {
     }
   }
 
+  computePublishedPackageUrl(distro: Distro, targetArchitecture: Arch, packageUrl: string): string {
+    const packageFileName = packageUrl.split('/').slice(-1);
+    const packageFolderVersion = this.mongodbVersion.split('.').slice(0, 2).join('.');
+    switch (distro) {
+      case Distro.Debian10:
+        return `${Barque.PPA_REPO_BASE_URL}/apt/debian/dists/buster/mongodb-org/${packageFolderVersion}/main/binary-${targetArchitecture}/${packageFileName}`;
+      case Distro.Ubuntu1804:
+        return `${Barque.PPA_REPO_BASE_URL}/apt/ubuntu/dists/bionic/mongodb-org/${packageFolderVersion}/multiverse/binary-${targetArchitecture}/${packageFileName}`;
+      case Distro.Ubuntu2004:
+        return `${Barque.PPA_REPO_BASE_URL}/apt/ubuntu/dists/focal/mongodb-org/${packageFolderVersion}/multiverse/binary-${targetArchitecture}/${packageFileName}`;
+      case Distro.Redhat80:
+        return `${Barque.PPA_REPO_BASE_URL}/yum/redhat/8/mongodb-org/${packageFolderVersion}/${targetArchitecture}/RPMS/${packageFileName}`;
+      default:
+        throw new Error(`Unsupported distro: ${distro}`);
+    }
+  }
+
+  /**
+   * Waits until the given packages are available under the specified URLs or throws an error if there
+   * are still remaining packages after the timeout.
+   *
+   * Note that the method will try all URLs at least once after an initial delay of `sleepTimeSeconds`.
+   */
+  async waitUntilPackagesAreAvailable(publishedPackageUrls: string[], timeoutSeconds: number, sleepTimeSeconds = 10): Promise<void> {
+    let remainingPackages = [...publishedPackageUrls];
+    const sleep = promisify(setTimeout);
+
+    const startMs = new Date().getTime();
+    const failOnTimeout = () => {
+      if (new Date().getTime() - startMs > timeoutSeconds * 1000) {
+        throw new Error(`Barque timed out - the following packages are still not available: ${remainingPackages.join(', ')}`);
+      }
+    };
+
+    while (remainingPackages.length) {
+      console.info(`Waiting for availability of:\n - ${remainingPackages.join('\n - ')}`);
+      await sleep(sleepTimeSeconds * 1000);
+
+      const promises = remainingPackages.map(async url => await fetch(url, {
+        method: 'HEAD'
+      }));
+      const responses = await Promise.all(promises);
+
+      const newRemainingPackages: string[] = [];
+      for (let i = 0; i < remainingPackages.length; i++) {
+        if (responses[i].status !== 200) {
+          newRemainingPackages.push(remainingPackages[i]);
+        }
+      }
+      remainingPackages = newRemainingPackages;
+
+      failOnTimeout();
+    }
+  }
+
   /**
    * Create a staging dir in /tmp to download the latest version of curator.
    *

packages/build/src/run-publish.spec.ts

@@ -81,7 +81,8 @@ describe('publish', () => {
     githubRepo = createStubRepo();
     mongoHomebrewRepo = createStubRepo();
     barque = createStubBarque({
-      releaseToBarque: sinon.stub().resolves(true)
+      releaseToBarque: sinon.stub().resolves(['package-url']),
+      waitUntilPackagesAreAvailable: sinon.stub().resolves()
     });
   });
 
@@ -188,6 +189,7 @@ describe('publish', () => {
         BuildVariant.Debian,
         'https://s3.amazonaws.com/mciuploads/project/v0.7.0-draft.42/mongosh_0.7.0_amd64.deb'
       );
+      expect(barque.waitUntilPackagesAreAvailable).to.have.been.called;
     });
 
     it('updates the download center config', async() => {

packages/build/src/run-publish.ts

@@ -92,11 +92,17 @@ async function publishArtifactsToBarque(
     BuildVariant.Debian,
     BuildVariant.Redhat
   ];
+
+  const publishedPackages: string[] = [];
   for await (const variant of variantsForBarque) {
     const tarballName = getTarballFile(variant, releaseVersion, packageName);
     const tarballUrl = getEvergreenArtifactUrl(project, mostRecentDraftTag, tarballName.path);
     console.info(`mongosh: Publishing ${variant} artifact to barque ${tarballUrl}`);
-    await barque.releaseToBarque(variant, tarballUrl);
+    const packageUrls = await barque.releaseToBarque(variant, tarballUrl);
+    publishedPackages.push(...packageUrls);
   }
+
+  await barque.waitUntilPackagesAreAvailable(publishedPackages, 300);
+
   console.info('mongosh: Submitting to barque complete');
 }