diff --git a/config/repo-config.yml b/config/repo-config.yml index 698a90a4d8..63a157282d 100644 --- a/config/repo-config.yml +++ b/config/repo-config.yml @@ -58,6 +58,17 @@ repos: # #################### + - name: debian92 + type: deb + code_name: "stretch" + bucket: repo.mongodb.org + edition: org + component: main + architectures: + - amd64 + repos: + - apt/debian/dists/stretch/mongodb-org + - name: debian10 type: deb code_name: "buster" @@ -77,6 +88,7 @@ repos: component: multiverse architectures: - amd64 + - arm64 repos: - apt/ubuntu/dists/bionic/mongodb-org @@ -88,13 +100,53 @@ repos: component: multiverse architectures: - amd64 + - arm64 repos: - apt/ubuntu/dists/focal/mongodb-org + - name: rhel70 + type: rpm + edition: org + bucket: repo.mongodb.org + architectures: + - x86_64 + repos: + - yum/redhat/7/mongodb-org + - yum/redhat/7Server/mongodb-org + - name: rhel80 type: rpm edition: org bucket: repo.mongodb.org + architectures: + - x86_64 + - aarch64 repos: - yum/redhat/8/mongodb-org - yum/redhat/8Server/mongodb-org + + - name: suse12 + type: rpm + edition: org + bucket: repo.mongodb.org + architectures: + - x86_64 + repos: + - zypper/suse/12/mongodb-org + + - name: suse15 + type: rpm + edition: org + bucket: repo.mongodb.org + repos: + - zypper/suse/15/mongodb-org + + - name: amazon2 + type: rpm + edition: org + bucket: repo.mongodb.org + architectures: + - arm64 + - x86_64 + repos: + - yum/amazon/2/mongodb-org diff --git a/package.json b/package.json index 1113bb87bf..09147fad95 100644 --- a/package.json +++ b/package.json @@ -84,8 +84,6 @@ "@types/bl": "^2.1.0", "@types/chai": "^4.2.5", "@types/download": "^6.2.4", - "@types/fs-extra": "^9.0.2", - "@types/gunzip-maybe": "^1.4.0", "@types/mocha": "^5.2.7", "@types/node": "^14.14.6", "@types/node-fetch": "^2.5.7", @@ -113,8 +111,6 @@ "eslint-config-mongodb-js": "^5.0.3", "eslint-plugin-mocha": "^6.2.2", "find-up": "^5.0.0", - "fs-extra": "^8.1.0", - "gunzip-maybe": "^1.4.2", "handlebars": "^4.7.7", "js-yaml": "^3.13.1", "karma": "^4.4.1", diff --git a/packages/build/src/barque.spec.ts b/packages/build/src/barque.spec.ts index 51e9ffc828..e8b4dbc222 100644 --- a/packages/build/src/barque.spec.ts +++ b/packages/build/src/barque.spec.ts @@ -1,11 +1,11 @@ import { expect } from 'chai'; -import fs from 'fs-extra'; +import { promises as fs } from 'fs'; import nock from 'nock'; import fetch from 'node-fetch'; import path from 'path'; import sinon from 'sinon'; import { URL } from 'url'; -import { Barque, LATEST_CURATOR } from './barque'; +import { Barque, LATEST_CURATOR, getReposAndArch } from './barque'; import { ALL_BUILD_VARIANTS, Config } from './config'; describe('Barque', () => { @@ -55,16 +55,19 @@ describe('Barque', () => { variant: 'debian-x64', url: 'https://s3.amazonaws.com/mciuploads/mongosh/5ed7ee5d8683818eb28d9d3b5c65837cde4a08f5/mongodb-mongosh_0.1.0_amd64.deb', publishedUrls: [ - `${Barque.PPA_REPO_BASE_URL}/apt/debian/dists/buster/mongodb-org/4.4/main/binary-amd64/mongodb-mongosh_0.1.0_amd64.deb`, `${Barque.PPA_REPO_BASE_URL}/apt/ubuntu/dists/bionic/mongodb-org/4.4/multiverse/binary-amd64/mongodb-mongosh_0.1.0_amd64.deb`, `${Barque.PPA_REPO_BASE_URL}/apt/ubuntu/dists/focal/mongodb-org/4.4/multiverse/binary-amd64/mongodb-mongosh_0.1.0_amd64.deb`, + `${Barque.PPA_REPO_BASE_URL}/apt/debian/dists/buster/mongodb-org/4.4/main/binary-amd64/mongodb-mongosh_0.1.0_amd64.deb`, + `${Barque.PPA_REPO_BASE_URL}/apt/debian/dists/stretch/mongodb-org/4.4/main/binary-amd64/mongodb-mongosh_0.1.0_amd64.deb` ] }, { variant: 'rhel-x64', url: 'https://s3.amazonaws.com/mciuploads/mongosh/5ed7ee5d8683818eb28d9d3b5c65837cde4a08f5/mongodb-mongosh-0.1.0.el7.x86_64.rpm', publishedUrls: [ + `${Barque.PPA_REPO_BASE_URL}/yum/redhat/7/mongodb-org/4.4/x86_64/RPMS/mongodb-mongosh-0.1.0.el7.x86_64.rpm`, `${Barque.PPA_REPO_BASE_URL}/yum/redhat/8/mongodb-org/4.4/x86_64/RPMS/mongodb-mongosh-0.1.0.el7.x86_64.rpm`, + `${Barque.PPA_REPO_BASE_URL}/yum/amazon/2/mongodb-org/4.4/x86_64/RPMS/mongodb-mongosh-0.1.0.el7.x86_64.rpm`, ] } ] as const).forEach(({ variant, url, publishedUrls }) => { @@ -93,7 +96,7 @@ describe('Barque', () => { try { await barque.releaseToBarque('debian-x64', debUrl); } catch (error) { - expect(error.message).to.include('Curator is unable to upload to barque'); + expect(error.message).to.include('Curator is unable to upload https://s3.amazonaws.com/mciuploads/mongosh/5ed7ee5d8683818eb28d9d3b5c65837cde4a08f5/mongodb-mongosh_0.1.0_amd64.deb,ubuntu1804,amd64 to barque'); expect(barque.createCuratorDir).to.have.been.called; expect(barque.extractLatestCurator).to.have.been.called; expect(barque.execCurator).to.have.been.called; @@ -115,56 +118,14 @@ describe('Barque', () => { }); }); - describe('getTargetDistro', () => { - it('determines distro for debian build variant', async() => { - const distro = barque.getTargetDistros('debian-x64'); - expect(distro).to.deep.equal(['debian10', 'ubuntu1804', 'ubuntu2004']); - }); - - it('determines distro for redhat build variant', async() => { - const distro = barque.getTargetDistros('rhel-x64'); - expect(distro).to.deep.equal(['rhel80']); - }); - - ALL_BUILD_VARIANTS - .filter(v => v !== 'debian-x64' && v !== 'rhel-x64') - .forEach(variant => { - it(`throws an error for ${variant}`, async() => { - try { - barque.getTargetDistros(variant); - } catch (e) { - expect(e.message).to.include(variant); - return; - } - expect.fail('Expected error'); - }); - }); - }); - - describe('getTargetArchitecture', () => { - it('determines arch for debian', async() => { - const distro = barque.getTargetArchitecture('debian-x64'); - expect(distro).to.be.equal('amd64'); - }); - - it('determines arch for redhat', async() => { - const distro = barque.getTargetArchitecture('rhel-x64'); - expect(distro).to.be.equal('x86_64'); - }); - - ALL_BUILD_VARIANTS - .filter(v => v !== 'debian-x64' && v !== 'rhel-x64') - .forEach(variant => { - it(`throws an error for ${variant}`, async() => { - try { - barque.getTargetArchitecture(variant); - } catch (e) { - expect(e.message).to.include(variant); - return; - } - expect.fail('Expected error'); - }); + describe('getReposAndArch', () => { + for (const variant of ALL_BUILD_VARIANTS) { + it(`returns results for ${variant}`, () => { + const result = getReposAndArch(variant); + expect(result.ppas).to.be.an('array'); + expect(result.arch).to.be.a('string'); }); + } }); describe('createCuratorDir', () => { diff --git a/packages/build/src/barque.ts b/packages/build/src/barque.ts index 32ad9b7bf7..855827c606 100644 --- a/packages/build/src/barque.ts +++ b/packages/build/src/barque.ts @@ -1,13 +1,13 @@ import childProcess from 'child_process'; -import fs from 'fs-extra'; -import gunzip from 'gunzip-maybe'; +import { promises as fs } from 'fs'; +import zlib from 'zlib'; import fetch from 'node-fetch'; import path from 'path'; import stream from 'stream'; import tar from 'tar-fs'; import tmp from 'tmp-promise'; import util, { promisify } from 'util'; -import { BuildVariant, Config, Platform } from './config'; +import { BuildVariant, getArch, getDistro, Config, getDebArchName, getRPMArchName, Platform } from './config'; const pipeline = util.promisify(stream.pipeline); const execFile = util.promisify(childProcess.execFile); @@ -19,26 +19,57 @@ export const LATEST_CURATOR = tmp.setGracefulCleanup(); /** - * Distro enum to be used when making a curator call. + * All the possible per-Linux-distro repositories that we publish to. */ -enum Distro { - Ubuntu1804 = 'ubuntu1804', - Ubuntu2004 = 'ubuntu2004', - Debian10 = 'debian10', - Redhat80 = 'rhel80' -} +type PPARepository = + 'ubuntu1804' | 'ubuntu2004' | 'debian92' | 'debian10' | + 'rhel70' | 'rhel80' | 'amazon2' | 'suse12' | 'suse15'; /** - * Target arch enum to be used when making a curator call. - * - * If we were to target a different arch for these distros, make sure - * config/repo-config.yml and packages/build/src/tarball.ts are changed accordingly. + * Return the full list of [distro, arch] combinations that we upload for + * a given build variant (where 'distro' refers to a distro in the package + * repository, e.g. Ubuntu 20.04). * - * This can be also moved to /config/build.conf.js in the future. + * /config/repo-config.yml needs to be kept in sync with this. */ -enum Arch { - Amd64 = 'amd64', - X86_64 = 'x86_64' +export function getReposAndArch(buildVariant: BuildVariant): { ppas: PPARepository[], arch: string } { + switch (getDistro(buildVariant)) { + case 'win32': + case 'win32msi': + case 'darwin': + case 'linux': + return { ppas: [], arch: '' }; + case 'debian': + return { + ppas: ['ubuntu1804', 'ubuntu2004', 'debian92', 'debian10'], + arch: getDebArchName(getArch(buildVariant)) + }; + case 'rhel': + if (getArch(buildVariant) === 'x64') { + return { + ppas: ['rhel70', 'rhel80', 'amazon2'], + arch: getRPMArchName(getArch(buildVariant)) + }; + } else if (getArch(buildVariant) === 'arm64') { + return { + ppas: ['rhel80'], + arch: getRPMArchName(getArch(buildVariant)) + }; + } + return { ppas: [], arch: '' }; + case 'suse': + return { + ppas: ['suse12', 'suse15'], + arch: getRPMArchName(getArch(buildVariant)) + }; + case 'amzn2': + return { + ppas: ['amazon2'], + arch: getRPMArchName(getArch(buildVariant)) + }; + default: + throw new Error(`Unknown build variant ${buildVariant}`); + } } export class Barque { @@ -78,25 +109,23 @@ export class Barque { const curatorDirPath = await this.createCuratorDir(); await this.extractLatestCurator(curatorDirPath); - const targetDistros = this.getTargetDistros(buildVariant); - const targetArchitecture = this.getTargetArchitecture(buildVariant); - const publishedPackageUrls: string[] = []; - for (const distro of targetDistros) { + const { ppas, arch } = getReposAndArch(buildVariant); + for (const ppa of ppas) { try { await this.execCurator( curatorDirPath, packageUrl, repoConfig, - distro, - targetArchitecture + ppa, + arch ); } catch (error) { console.error('Curator failed', error); - throw new Error(`Curator is unable to upload to barque ${error}`); + throw new Error(`Curator is unable to upload ${packageUrl},${ppa},${arch} to barque ${error}`); } - publishedPackageUrls.push(this.computePublishedPackageUrl(distro, targetArchitecture, packageUrl)); + publishedPackageUrls.push(this.computePublishedPackageUrl(ppa, arch, packageUrl)); } return publishedPackageUrls; } @@ -105,8 +134,8 @@ export class Barque { curatorDirPath: string, packageUrl: string, repoConfig: string, - distro: Distro, - architecture: Arch + ppa: PPARepository, + architecture: string ): Promise { return await execFile( `${curatorDirPath}/curator`, [ @@ -114,7 +143,7 @@ export class Barque { 'repo', 'submit', '--service', 'https://barque.corp.mongodb.com', '--config', repoConfig, - '--distro', distro, + '--distro', ppa, '--arch', architecture, '--edition', this.mongodbEdition, '--version', this.mongodbVersion, @@ -130,48 +159,23 @@ export class Barque { }); } - getTargetArchitecture(variant: BuildVariant): Arch { - switch (variant) { - case 'debian-x64': - return Arch.Amd64; - case 'rhel-x64': - return Arch.X86_64; - default: - throw new Error('Unsupported variant for Barque publishing: ' + variant); - } - } - - getTargetDistros(variant: BuildVariant): Distro[] { - switch (variant) { - case 'debian-x64': - return [ - Distro.Debian10, - Distro.Ubuntu1804, - Distro.Ubuntu2004 - ]; - case 'rhel-x64': - return [ - Distro.Redhat80 - ]; - default: - throw new Error('Unsupported variant for Barque publishing: ' + variant); - } - } - - computePublishedPackageUrl(distro: Distro, targetArchitecture: Arch, packageUrl: string): string { + computePublishedPackageUrl(ppa: PPARepository, targetArchitecture: string, packageUrl: string): string { const packageFileName = packageUrl.split('/').slice(-1); const packageFolderVersion = this.mongodbVersion.split('.').slice(0, 2).join('.'); - switch (distro) { - case Distro.Debian10: - return `${Barque.PPA_REPO_BASE_URL}/apt/debian/dists/buster/mongodb-org/${packageFolderVersion}/main/binary-${targetArchitecture}/${packageFileName}`; - case Distro.Ubuntu1804: - return `${Barque.PPA_REPO_BASE_URL}/apt/ubuntu/dists/bionic/mongodb-org/${packageFolderVersion}/multiverse/binary-${targetArchitecture}/${packageFileName}`; - case Distro.Ubuntu2004: - return `${Barque.PPA_REPO_BASE_URL}/apt/ubuntu/dists/focal/mongodb-org/${packageFolderVersion}/multiverse/binary-${targetArchitecture}/${packageFileName}`; - case Distro.Redhat80: - return `${Barque.PPA_REPO_BASE_URL}/yum/redhat/8/mongodb-org/${packageFolderVersion}/${targetArchitecture}/RPMS/${packageFileName}`; + switch (ppa) { + /* eslint-disable no-multi-spaces */ + case 'ubuntu1804': return `${Barque.PPA_REPO_BASE_URL}/apt/ubuntu/dists/bionic/mongodb-org/${packageFolderVersion}/multiverse/binary-${targetArchitecture}/${packageFileName}`; + case 'ubuntu2004': return `${Barque.PPA_REPO_BASE_URL}/apt/ubuntu/dists/focal/mongodb-org/${packageFolderVersion}/multiverse/binary-${targetArchitecture}/${packageFileName}`; + case 'debian92': return `${Barque.PPA_REPO_BASE_URL}/apt/debian/dists/buster/mongodb-org/${packageFolderVersion}/main/binary-${targetArchitecture}/${packageFileName}`; + case 'debian10': return `${Barque.PPA_REPO_BASE_URL}/apt/debian/dists/stretch/mongodb-org/${packageFolderVersion}/main/binary-${targetArchitecture}/${packageFileName}`; + case 'rhel70': return `${Barque.PPA_REPO_BASE_URL}/yum/redhat/7/mongodb-org/${packageFolderVersion}/${targetArchitecture}/RPMS/${packageFileName}`; + case 'rhel80': return `${Barque.PPA_REPO_BASE_URL}/yum/redhat/8/mongodb-org/${packageFolderVersion}/${targetArchitecture}/RPMS/${packageFileName}`; + case 'amazon2': return `${Barque.PPA_REPO_BASE_URL}/yum/amazon/2/mongodb-org/${packageFolderVersion}/${targetArchitecture}/RPMS/${packageFileName}`; + case 'suse12': return `${Barque.PPA_REPO_BASE_URL}/zypper/suse/12/mongodb-org/${packageFolderVersion}/${targetArchitecture}/RPMS/${packageFileName}`; + case 'suse15': return `${Barque.PPA_REPO_BASE_URL}/zypper/suse/15/mongodb-org/${packageFolderVersion}/${targetArchitecture}/RPMS/${packageFileName}`; + /* eslint-enable no-multi-spaces */ default: - throw new Error(`Unsupported distro: ${distro}`); + throw new Error(`Unsupported PPA: ${ppa}`); } } @@ -220,7 +224,7 @@ export class Barque { */ async createCuratorDir(): Promise { const dir = await tmp.dir({ prefix: 'curator-', unsafeCleanup: true }); - fs.ensureDir(dir.path, { mode: 0o755 }); + await fs.mkdir(dir.path, { mode: 0o755, recursive: true }); return dir.path; } @@ -243,7 +247,7 @@ export class Barque { if (response.ok) { return pipeline( response.body, - gunzip(), + zlib.createGunzip(), tar.extract(dest) ); } diff --git a/packages/build/src/run-publish.spec.ts b/packages/build/src/run-publish.spec.ts index 6ebed49e42..3ee19b091e 100644 --- a/packages/build/src/run-publish.spec.ts +++ b/packages/build/src/run-publish.spec.ts @@ -178,7 +178,7 @@ describe('publish', () => { shouldDoPublicRelease ); - expect(barque.releaseToBarque).to.have.been.callCount(2); + expect(barque.releaseToBarque).to.have.been.callCount(15); expect(barque.releaseToBarque).to.have.been.calledWith( 'rhel-x64', 'https://s3.amazonaws.com/mciuploads/project/v0.7.0-draft.42/mongodb-mongosh-0.7.0.el7.x86_64.rpm' @@ -187,6 +187,10 @@ describe('publish', () => { 'debian-x64', 'https://s3.amazonaws.com/mciuploads/project/v0.7.0-draft.42/mongodb-mongosh_0.7.0_amd64.deb' ); + expect(barque.releaseToBarque).to.have.been.calledWith( + 'amzn2-arm64', + 'https://s3.amazonaws.com/mciuploads/project/v0.7.0-draft.42/mongodb-mongosh-0.7.0.amzn2.aarch64.rpm' + ); expect(barque.waitUntilPackagesAreAvailable).to.have.been.called; }); diff --git a/packages/build/src/run-publish.ts b/packages/build/src/run-publish.ts index f2953d926c..fa44f42a2e 100644 --- a/packages/build/src/run-publish.ts +++ b/packages/build/src/run-publish.ts @@ -1,7 +1,7 @@ import type { writeAnalyticsConfig as writeAnalyticsConfigType } from './analytics'; import { Barque } from './barque'; import { - BuildVariant, + ALL_BUILD_VARIANTS, Config, getReleaseVersionFromTag, shouldDoPublicRelease as shouldDoPublicReleaseFn @@ -87,13 +87,8 @@ async function publishArtifactsToBarque( packageInformation: PackageInformation, getEvergreenArtifactUrl: typeof getArtifactUrlFn ): Promise { - const variantsForBarque: BuildVariant[] = [ - 'debian-x64', - 'rhel-x64' - ]; - const publishedPackages: string[] = []; - for await (const variant of variantsForBarque) { + for await (const variant of ALL_BUILD_VARIANTS) { const packageFile = getPackageFile(variant, { ...packageInformation, metadata: { @@ -102,8 +97,11 @@ async function publishArtifactsToBarque( } }); const packageUrl = getEvergreenArtifactUrl(project, mostRecentDraftTag, packageFile.path); - console.info(`mongosh: Publishing ${variant} artifact to barque ${packageUrl}`); + console.info(`mongosh: Considering publishing ${variant} artifact to barque ${packageUrl}`); const packageUrls = await barque.releaseToBarque(variant, packageUrl); + for (const url of packageUrls) { + console.info(` -> ${url}`); + } publishedPackages.push(...packageUrls); }