diff --git a/config/repo-config.yml b/config/repo-config.yml new file mode 100644 index 0000000000..95210522ad --- /dev/null +++ b/config/repo-config.yml @@ -0,0 +1,89 @@ +services: + notary_url: "http://notary-service.build.10gen.cc:5000" + +templates: + deb: + org: | + Origin: mongodb + Label: mongodb + Suite: {{ .CodeName }} + Codename: {{ .CodeName }}/mongodb-org + Architectures: {{ .Architectures }} + Components: {{ .Component }} + Description: MongoDB packages + enterprise: | + Origin: mongodb + Label: mongodb + Suite: {{ .CodeName }} + Codename: {{ .CodeName }}/mongodb-enterprise + Architectures: {{ .Architectures }} + Components: {{ .Component }} + Description: MongoDB packages + index_page: | + + + + {{ .Title }} + + + + + + + {{ range $fn := .Files }} + + {{ end }} + + +
+

{{ .Title }}

+
+
+
+ Parent Directory +
+ {{ $fn }} +
+
+
+
{{ .RepoName }}
+
+ + +repos: + +#################### +# +# Community Repos: +# +#################### + + - name: debian10 + type: deb + code_name: "buster" + bucket: repo.mongodb.org + edition: org + component: main + architectures: + - amd64 + repos: + - apt/debian/dists/buster/mongodb-org + + - name: ubuntu1804 + type: deb + code_name: "bionic" + edition: org + bucket: repo.mongodb.org + component: multiverse + architectures: + - amd64 + repos: + - apt/ubuntu/dists/bionic/mongodb-org + + - name: rhel80 + type: rpm + edition: org + bucket: repo.mongodb.org + repos: + - yum/redhat/8/mongodb-org + - yum/redhat/8Server/mongodb-org \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 527c35db5c..16f2a4bb76 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10216,6 +10216,37 @@ "integrity": "sha512-qBr4OuELkhPenW6goKVXiv47US3clb3/IbuWF9KNKEijAy9oeHxU9IgzjvJhHkUzhaj7rOUD7+YGWqUjLp5oSA==", "dev": true }, + "gunzip-maybe": { + "version": "1.4.2", + "resolved": "https://registry.npmjs.org/gunzip-maybe/-/gunzip-maybe-1.4.2.tgz", + "integrity": "sha512-4haO1M4mLO91PW57BMsDFf75UmwoRX0GkdD+Faw+Lr+r/OZrOCS0pIBwOL1xCKQqnQzbNFGgK2V2CpBUPeFNTw==", + "dev": true, + "requires": { + "browserify-zlib": "^0.1.4", + "is-deflate": "^1.0.0", + "is-gzip": "^1.0.0", + "peek-stream": "^1.1.0", + "pumpify": "^1.3.3", + "through2": "^2.0.3" + }, + "dependencies": { + "browserify-zlib": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/browserify-zlib/-/browserify-zlib-0.1.4.tgz", + "integrity": "sha1-uzX4pRn2AOD6a4SFJByXnQFB+y0=", + "dev": true, + "requires": { + "pako": "~0.2.0" + } + }, + "pako": { + "version": "0.2.9", + "resolved": "https://registry.npmjs.org/pako/-/pako-0.2.9.tgz", + "integrity": "sha1-8/dSL073gjSNqBYbrZ7P1Rv4OnU=", + "dev": true + } + } + }, "handlebars": { "version": "4.7.6", "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.7.6.tgz", @@ -11049,6 +11080,12 @@ "integrity": "sha512-USlDT524woQ08aoZFzh3/Z6ch9Y/EWXEHQ/AaRN0SkKq4t2Jw2R2339tSXmwuVoY7LLlBCbOIlx2myP/L5zk0g==", "dev": true }, + "is-deflate": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-deflate/-/is-deflate-1.0.0.tgz", + "integrity": "sha1-yGKQHDwWH7CdrHzcfnhPgOmPLxQ=", + "dev": true + }, "is-descriptor": { "version": "0.1.6", "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-0.1.6.tgz", @@ -11116,6 +11153,12 @@ "is-extglob": "^2.1.1" } }, + "is-gzip": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-gzip/-/is-gzip-1.0.0.tgz", + "integrity": "sha1-bKiwe5nHeZgCWQDlVc7Y7YCHmoM=", + "dev": true + }, "is-html": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/is-html/-/is-html-1.1.0.tgz", @@ -14144,9 +14187,9 @@ } }, "node-fetch": { - "version": "2.6.0", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.0.tgz", - "integrity": "sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA==" + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz", + "integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==" }, "node-fetch-npm": { "version": "2.0.4", @@ -15308,6 +15351,17 @@ "sha.js": "^2.4.8" } }, + "peek-stream": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/peek-stream/-/peek-stream-1.1.3.tgz", + "integrity": "sha512-FhJ+YbOSBb9/rIl2ZeE/QHEsWn7PqNYt8ARAY3kIgNGOk13g9FGyIY6JIl/xB/3TFRVoTv5as0l11weORrTekA==", + "dev": true, + "requires": { + "buffer-from": "^1.0.0", + "duplexify": "^3.5.0", + "through2": "^2.0.3" + } + }, "pend": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", @@ -18750,6 +18804,65 @@ } } }, + "tar-fs": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.0.tgz", + "integrity": "sha512-9uW5iDvrIMCVpvasdFHW0wJPez0K4JnMZtsuIeDI7HyMGJNxmDZDOCQROr7lXyS+iL/QMpj07qcjGYTSdRFXUg==", + "dev": true, + "requires": { + "chownr": "^1.1.1", + "mkdirp-classic": "^0.5.2", + "pump": "^3.0.0", + "tar-stream": "^2.0.0" + }, + "dependencies": { + "bl": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/bl/-/bl-4.0.3.tgz", + "integrity": "sha512-fs4G6/Hu4/EE+F75J8DuN/0IpQqNjAdC7aEQv7Qt8MHGUH7Ckv2MwTEEeN9QehD0pfIDkMI1bkHYkKy7xHyKIg==", + "dev": true, + "requires": { + "buffer": "^5.5.0", + "inherits": "^2.0.4", + "readable-stream": "^3.4.0" + } + }, + "buffer": { + "version": "5.6.0", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.6.0.tgz", + "integrity": "sha512-/gDYp/UtU0eA1ys8bOs9J6a+E/KWIY+DZ+Q2WESNUA0jFRsJOc0SNUO6xJ5SGA1xueg3NL65W6s+NY5l9cunuw==", + "dev": true, + "requires": { + "base64-js": "^1.0.2", + "ieee754": "^1.1.4" + } + }, + "readable-stream": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", + "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", + "dev": true, + "requires": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + } + }, + "tar-stream": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.1.4.tgz", + "integrity": "sha512-o3pS2zlG4gxr67GmFYBLlq+dM8gyRGUOvsrHclSkvtVtQbjV0s/+ZE8OpICbaj8clrX3tjeHngYGP7rweaBnuw==", + "dev": true, + "requires": { + "bl": "^4.0.3", + "end-of-stream": "^1.4.1", + "fs-constants": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1" + } + } + } + }, "tar-stream": { "version": "1.6.2", "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-1.6.2.tgz", diff --git a/package.json b/package.json index 496fa9bd4c..b7401219d7 100644 --- a/package.json +++ b/package.json @@ -85,6 +85,7 @@ "eslint-config-mongodb-js": "^5.0.3", "eslint-plugin-mocha": "^6.2.2", "fs-extra": "^8.1.0", + "gunzip-maybe": "^1.4.2", "handlebars": "^4.7.6", "js-yaml": "^3.13.1", "karma": "^4.4.1", @@ -98,6 +99,7 @@ "mongodb-js-precommit": "^2.0.0", "mongodb-runner": "^4.7.5", "node-codesign": "github:addaleax/node-codesign", + "node-fetch": "^2.6.1", "parcel-bundler": "^1.12.4", "pkg": "^4.4.3", "pkg-deb": "^1.1.1", @@ -107,6 +109,8 @@ "sinon": "^7.5.0", "sinon-chai": "^3.4.0", "tar": "^6.0.1", + "tar-fs": "^2.1.0", + "tmp-promise": "^3.0.2", "ts-node": "^8.5.2", "ts-sinon": "^1.2.0", "typescript": "^3.9.7", diff --git a/packages/build/README.md b/packages/build/README.md index 4f70b4955b..3b7ea696a5 100644 --- a/packages/build/README.md +++ b/packages/build/README.md @@ -10,25 +10,33 @@ root of the project. Current build and release flow is as follows: -- `npm run evergreen-release package - - A commit triggers an evergreen build based on currently available build - variants: MacOS, Windows, Linux, Debian, and RedHat. - - MacOS, Linux and Windows run three tasks: check, test, and release. Debian and - Redhat run two tasks: check and release. Debian and Redhat also depend on - tests to pass on Linux. - - Identical bundle and binary are built on all five variants. - - Each variant creates its own tarball (`.zip`, `.tgz`, `.deb`, `.rpm`). Type of - tarball is determined by the current build variant. - - Each variant uploads its own tarball to Evergreen’s AWS. - - MacOS build variant uploads config file with information about the new version - for each platform to Downloads Centre. This only happens on a tagged commit. - - MacOS build variant creates a github release. This only happens on a tagged - commit. - - The five build variants run in parallel. -- `npm run evergreen-release publish` - - All the previous build steps succeeded. - - A separate MacOS build variant (darwin_publish_release) uploads config file with information about the new version for each platform to Downloads Centre. This only happens on a tagged commit. - - A separate MacOS build variant (darwin_publish_release) promotes the draft github release to public. This only happens on a tagged commit. +### `npm run evergreen-release package` +- A commit triggers an evergreen build based on currently available build + variants: MacOS, Windows, Linux, Debian, and RedHat. +- MacOS, Linux and Windows run three tasks: check, test, and release. Debian and + Redhat run two tasks: check and release. Debian and Redhat also depend on + tests to pass on Linux. +- Identical bundle and binary are built on all five variants. +- Each variant creates its own tarball (`.zip`, `.tgz`, `.deb`, `.rpm`). Type of + tarball is determined by the current build variant. +- Each variant uploads its own tarball to Evergreen’s AWS. +- Linux build variants upload their artifacts to `barque` using + [`curator`](https://github.com/mongodb/curator) to be used with MongoDB's PPA. The uploaded packages can be found under the following URLs: + 1. Ubuntu: https://repo.mongodb.org/apt/ubuntu/dists/bionic/mongodb-org/4.4/multiverse/binary-amd64/ + 2. Redhat: https://repo.mongodb.org/yum/redhat/8Server/mongodb-org/4.4/x86_64/RPMS/ + 3. Debian: https://repo.mongodb.org/apt/debian/dists/buster/mongodb-org/4.4/main/binary-amd64/ +- MacOS build variant uploads config file with information about the new version + for each platform to Downloads Centre. This only happens on a tagged commit. +- MacOS build variant creates a github release. This only happens on a tagged + commit. +- The five build variants run in parallel. +### `npm run evergreen-release publish` +- All the previous build steps succeeded. +- A separate MacOS build variant (darwin_publish_release) uploads config file + with information about the new version for each platform to Downloads Centre. +This only happens on a tagged commit. +- A separate MacOS build variant (darwin_publish_release) promotes the draft + github release to public. This only happens on a tagged commit. ![build flow][build-img] diff --git a/packages/build/src/barque.spec.ts b/packages/build/src/barque.spec.ts new file mode 100644 index 0000000000..aeaa458b17 --- /dev/null +++ b/packages/build/src/barque.spec.ts @@ -0,0 +1,183 @@ +import { Barque } from './barque'; +import { expect } from 'chai'; +import Config from './config'; +import sinon from 'sinon'; +import fs from 'fs-extra'; +import path from 'path'; + +describe('Barque', () => { + let barque: Barque; + let config: Config; + + beforeEach(() => { + config = { + version: 'version', + bundleId: 'bundleId', + input: 'input', + execInput: 'execInput', + outputDir: 'outputDir', + analyticsConfig: 'analyticsConfig', + project: 'project', + revision: 'revision', + branch: 'branch', + evgAwsKey: 'evgAwsKey', + evgAwsSecret: 'evgAwsSecret', + downloadCenterAwsKey: 'downloadCenterAwsKey', + downloadCenterAwsSecret: 'downloadCenterAwsSecret', + githubToken: 'githubToken', + segmentKey: 'segmentKey', + rootDir: '../../../', + appleUser: 'appleUser', + applePassword: 'applePassword', + appleAppIdentity: 'appleAppIdentity', + isCi: true, + platform: 'linux', + buildVariant: 'linux', + repo: { + owner: 'owner', + repo: 'repo', + } + }; + + barque = new Barque(config); + }); + + describe('.releaseToBarque', () => { + context('platform is linux', () => { + it('execCurator function succeeds', async() => { + barque.execCurator = sinon.stub().returns(Promise.resolve(true)); + barque.createCuratorDir = sinon.stub().returns(Promise.resolve('./')); + barque.extractLatestCurator = sinon.stub().returns(Promise.resolve(true)); + + const tarballURL = 'https://s3.amazonaws.com/mciuploads/mongosh/5ed7ee5d8683818eb28d9d3b5c65837cde4a08f5/mongosh-0.1.0-linux.tgz'; + + let err; + + try { + await barque.releaseToBarque(tarballURL); + } catch (error) { + err = error; + } + expect(err).to.be.undefined; + expect(barque.createCuratorDir).to.have.been.called; + expect(barque.extractLatestCurator).to.have.been.called; + expect(barque.execCurator).to.have.been.called; + }); + + it('execCurator function fails', async() => { + barque.execCurator = sinon.stub().rejects(new Error('error')); + barque.createCuratorDir = sinon.stub().returns(Promise.resolve('./')); + barque.extractLatestCurator = sinon.stub().returns(Promise.resolve(true)); + + const tarballURL = 'https://s3.amazonaws.com/mciuploads/mongosh/5ed7ee5d8683818eb28d9d3b5c65837cde4a08f5/mongosh-0.1.0-linux.tgz'; + + let err; + + try { + await barque.releaseToBarque(tarballURL); + } catch (error) { + err = error; + } + + expect(err).to.not.be.undefined; + expect(err.message).to.include('Curator is unable to upload to barque'); + expect(barque.createCuratorDir).to.have.been.called; + expect(barque.extractLatestCurator).to.have.been.called; + expect(barque.execCurator).to.have.been.called; + }); + }); + + it('platform is not linux', async() => { + config.platform = 'macos'; + barque = new Barque(config); + + barque.execCurator = sinon.stub().returns(Promise.resolve(true)); + barque.createCuratorDir = sinon.stub().returns(Promise.resolve('./')); + barque.extractLatestCurator = sinon.stub().returns(Promise.resolve(true)); + + const tarballURL = 'https://s3.amazonaws.com/mciuploads/mongosh/5ed7ee5d8683818eb28d9d3b5c65837cde4a08f5/mongosh-0.1.0-linux.tgz'; + + await barque.releaseToBarque(tarballURL); + expect(barque.createCuratorDir).to.have.been.called; + expect(barque.extractLatestCurator).to.have.been.called; + expect(barque.execCurator).to.not.have.been.called; + }); + }); + + describe('.determineDistro', () => { + it('determines distro for ubuntu', async() => { + const distro = barque.determineDistro('linux'); + expect(distro).to.be.equal('ubuntu1804'); + }); + + it('determines distro for debian', async() => { + const distro = barque.determineDistro('debian'); + expect(distro).to.be.equal('debian10'); + }); + + it('determines distro for redhat', async() => { + const distro = barque.determineDistro('rhel'); + expect(distro).to.be.equal('rhel80'); + }); + + it('defaults to ubuntu distro', async() => { + const distro = barque.determineDistro('redhat'); + expect(distro).to.be.equal('ubuntu1804'); + }); + }); + + describe('.determineArch', () => { + it('determines arch for ubuntu', async() => { + const distro = barque.determineArch('linux'); + expect(distro).to.be.equal('amd64'); + }); + + it('determines arch for debian', async() => { + const distro = barque.determineArch('debian'); + expect(distro).to.be.equal('amd64'); + }); + + it('determines arch for redhat', async() => { + const distro = barque.determineArch('rhel'); + expect(distro).to.be.equal('x86_64'); + }); + + it('defaults to ubuntu arch', async() => { + const distro = barque.determineArch('redhat'); + expect(distro).to.be.equal('amd64'); + }); + }); + + describe('.createCuratorDir', () => { + it('creates tmp directory that exists', async() => { + const curatorDirPath = await barque.createCuratorDir(); + + let accessErr + try { + await fs.access(curatorDirPath) + } catch (e) { + accessErr = e + } + // eslint-disable-next-line + expect(accessErr).to.be.undefined + }); + }); + + describe('.extractLatestCurator', () => { + it('extractss latest curator to tmp directory', async() => { + const curatorDirPath = await barque.createCuratorDir(); + const curatorPath = path.join(curatorDirPath, 'curator'); + + await barque.extractLatestCurator(curatorDirPath); + + let accessErr + try { + await fs.access(curatorPath) + } catch (e) { + accessErr = e + } + // eslint-disable-next-line + expect(accessErr).to.be.undefined + }); + }); +}); diff --git a/packages/build/src/barque.ts b/packages/build/src/barque.ts new file mode 100644 index 0000000000..702bcff802 --- /dev/null +++ b/packages/build/src/barque.ts @@ -0,0 +1,192 @@ +import BuildVariant from './build-variant'; +import child_process from 'child_process'; +import gunzip from 'gunzip-maybe'; +import Platform from './platform'; +import fetch from 'node-fetch'; +import tmp from 'tmp-promise'; +import Config from './config'; +import stream from 'stream'; +import fs from 'fs-extra'; +import tar from 'tar-fs'; +import util from 'util'; +import path from 'path'; + +const pipeline = util.promisify(stream.pipeline); +const execFile = util.promisify(child_process.execFile); + +const LATEST_CURATOR = + 'https://s3.amazonaws.com/boxes.10gen.com/build/curator/curator-dist-ubuntu1604-latest.tar.gz'; + +// make sure everything written in /tmp is cleared if an uncaught exception occurs +tmp.setGracefulCleanup() + +/** + * Distro enum to be used when making a curator call. + */ +enum Distro { + Ubuntu = 'ubuntu1804', + Debian = 'debian10', + Redhat = 'rhel80' +} + +/** + * Target arch enum to be used when making a curator call. + * + * If we were to target a different arch for these distros, make sure + * config/repo-config.yml and packages/build/src/tarball.ts are changed accordingly. + * + * This can be also moved to /config/build.conf.js in the future. + */ +enum Arch { + Ubuntu = 'amd64', + Debian = 'amd64', + Redhat = 'x86_64', +} + +export class Barque { + private config: Config; + private mongodbEdition: string; + private mongodbVersion: string; + + constructor(config: Config) { + this.config = config; + // hard code mongodb edition to 'org' for now + this.mongodbEdition = 'org'; + // linux mongodb versions to release to. This should perhaps be an array of + // [4.3.0, 4.4.0], like mongo-tools + this.mongodbVersion = '4.4.0'; + } + + /** + * Upload current package to barque, MongoDB's PPA for linux distros. + * + * @param {string} tarballURL- The uploaded to Evergreen tarball URL. + * @param {Config} config - Config object. + * + * @returns {Promise} The promise. + */ + async releaseToBarque(tarballURL: string): Promise { + const repoConfig = path.join(this.config.rootDir, 'config', 'repo-config.yml'); + const curatorDirPath = await this.createCuratorDir(); + await this.extractLatestCurator(curatorDirPath); + + if (this.config.platform === Platform.Linux) { + try { + await this.execCurator(curatorDirPath, tarballURL, repoConfig); + } catch (error) { + throw new Error(`Curator is unable to upload to barque ${error}`); + } + } + + return; + } + + /** + * Run the child_process.exec to run curator command. + * + * @param {string} curatorDir - Path to freshly downloaded curator. + * @param {string} tarballURL- The uploaded to Evergreen tarball URL. + * @param {string} repoConfig - Path to repo-config.yml used to uplaod assets + * to appropriate distros. + * + * @returns {Promise} The promise. + */ + async execCurator( + curatorDirPath: string, + tarballURL: string, + repoConfig: string): Promise { + return await execFile( + `${curatorDirPath}/curator`, [ + '--level', 'debug', + 'repo', 'submit', + '--service', 'https://barque.corp.mongodb.com', + '--config', repoConfig, + '--distro', this.determineDistro(this.config.buildVariant), + '--arch', this.determineArch(this.config.buildVariant), + '--edition', this.mongodbEdition, + '--version', this.mongodbVersion, + '--packages', tarballURL + ], { + // curator looks for these options in env + env: { + NOTARY_KEY_NAME: 'server-4.4', + NOTARY_TOKEN: process.env.SIGNING_AUTH_TOKEN_44, + BARQUE_API_KEY: process.env.BARQUE_API_KEY, + BARQUE_USERNAME: process.env.BARQUE_USERNAME + } + }); + } + + /** + * Determine the current arch to be passed on to curator given current build + * variant. + * + * @param {string} variant - Current build variant. + * + * @returns {string} Arch to be passed as an argument to curator + */ + determineArch(variant: string): string { + // we can't use distro_id from evergreen's env variables, since those + // sometimes come with other attributes like -test -large -small, and are not + // valid distros for barque. + if (variant === BuildVariant.Linux) return Arch.Ubuntu; + if (variant === BuildVariant.Debian) return Arch.Debian; + if (variant === BuildVariant.Redhat) return Arch.Redhat; + + return Arch.Ubuntu; + } + /** + * Determine the current distro to be passed on to curator given current build + * variant. + * + * @param {string} variant - Current build variant. + * + * @returns {string} Distro to be passed as an argument to curator + */ + determineDistro(variant: string): string { + // we can't use distro_id from evergreen's env variables, since those + // sometimes come with other attributes like -test -large -small, and are not + // valid distros for barque. + if (variant === BuildVariant.Linux) return Distro.Ubuntu; + if (variant === BuildVariant.Debian) return Distro.Debian; + if (variant === BuildVariant.Redhat) return Distro.Redhat; + + return Distro.Ubuntu; + } + + /** + * Create a staging dir in /tmp to download the latest version of curator. + * + * @returns {Promise} Staging directory path. + */ + async createCuratorDir(): Promise { + const dir = await tmp.dir({ prefix: 'curator-', unsafeCleanup: true }); + fs.ensureDir(dir.path, '0755'); + + return dir.path; + } + + /** + * Fetch, unzip, and un-tar the latest version of curator. Then write it to + * previoiusly created /tmp staging directory. + * + * @param {string} dest - Destination to write the un-packaged curator + * executable. + * + * @returns {Promise} Written binary in the given location. + * + * Debian and Ubuntu Build Variants' curator errors out on `execFile`, to make + * sure this functionality works as expected, download the latest curator. + * + */ + async extractLatestCurator(dest: string): Promise { + const response = await fetch(LATEST_CURATOR) + if (response.ok) { + return pipeline( + response.body, + gunzip(), + tar.extract(dest) + ); + } + } +} diff --git a/packages/build/src/build-and-upload.spec.ts b/packages/build/src/build-and-upload.spec.ts index b516db2bdd..8e4cd9d784 100644 --- a/packages/build/src/build-and-upload.spec.ts +++ b/packages/build/src/build-and-upload.spec.ts @@ -1,9 +1,10 @@ -import { GithubRepo } from './github-repo'; import buildAndUpload from './build-and-upload'; +import { GithubRepo } from './github-repo'; +import { TarballFile } from './tarball'; import chai, { expect } from 'chai'; +import { Barque } from './barque'; import Config from './config'; import sinon from 'ts-sinon'; -import { TarballFile } from './tarball'; chai.use(require('sinon-chai')); @@ -11,12 +12,17 @@ function createStubRepo(overrides?: any): GithubRepo { return sinon.createStubInstance(GithubRepo, overrides) as unknown as GithubRepo; } +function createStubBarque(overrides?: any): Barque { + return sinon.createStubInstance(Barque, overrides) as unknown as Barque; +} + describe('buildAndRelease', () => { let config: Config; let tarballFile: TarballFile; let compileAndZipExecutable: (Config) => Promise; let uploadToEvergreen: (artifact: string, awsKey: string, awsSecret: string, project: string, revision: string) => Promise; let uploadToDownloadCenter: (artifact: string, awsKey: string, awsSecret: string) => Promise; + let barque: Barque; let githubRepo: GithubRepo; beforeEach(() => { @@ -50,20 +56,26 @@ describe('buildAndRelease', () => { tarballFile = { path: 'path', contentType: 'application/gzip' }; compileAndZipExecutable = sinon.stub().resolves(tarballFile); + barque = createStubBarque(); uploadToEvergreen = sinon.spy(); uploadToDownloadCenter = sinon.spy(); githubRepo = createStubRepo(); }); [true, false].forEach((isPublicRelease) => { - it(`uploads the artifact to evergreen if is ${isPublicRelease ? 'a' : 'not a'} public release`, async() => { + it(`uploads the artifact to evergreen if is ${isPublicRelease ? 'a' : 'not a'} public release`, async () => { githubRepo = createStubRepo({ shouldDoPublicRelease: sinon.stub().returns(Promise.resolve(isPublicRelease)) }); + barque = createStubBarque({ + releaseToBarque: sinon.stub().returns(Promise.resolve(true)) + }); + await buildAndUpload( config, githubRepo, + barque, compileAndZipExecutable, uploadToEvergreen, uploadToDownloadCenter @@ -79,14 +91,19 @@ describe('buildAndRelease', () => { }); }); - it('releases to github if a public release', async() => { + it('releases to github if a public release', async () => { githubRepo = createStubRepo({ - shouldDoPublicRelease: sinon.stub().returns(Promise.resolve(true)) + shouldDoPublicRelease: sinon.stub().resolves(true) + }); + + barque = createStubBarque({ + releaseToBarque: sinon.stub().resolves(true) }); await buildAndUpload( config, githubRepo, + barque, compileAndZipExecutable, uploadToEvergreen, uploadToDownloadCenter @@ -99,15 +116,83 @@ describe('buildAndRelease', () => { ); }); - it('releases to downloads centre if a public release', async() => { + it('does not release to github if not a public release', async () => { + githubRepo = createStubRepo({ + shouldDoPublicRelease: sinon.stub().resolves(false) + }); + + barque = createStubBarque({ + releaseToBarque: sinon.stub().resolves(true) + }); + + await buildAndUpload( + config, + githubRepo, + barque, + compileAndZipExecutable, + uploadToEvergreen, + uploadToDownloadCenter + ); + + expect(uploadToDownloadCenter).to.not.have.been.called; + }); + + it('releases to barque if a public release', async () => { + githubRepo = createStubRepo({ + shouldDoPublicRelease: sinon.stub().returns(Promise.resolve(true)) + }); + + barque = createStubBarque({ + releaseToBarque: sinon.stub().returns(Promise.resolve(true)) + }); + + await buildAndUpload( + config, + githubRepo, + barque, + compileAndZipExecutable, + uploadToEvergreen, + uploadToDownloadCenter + ); + + expect(barque.releaseToBarque).to.have.been.called; + }); + + it('does not releases to barque if not a public release', async () => { githubRepo = createStubRepo({ - shouldDoPublicRelease: sinon.stub().returns(Promise.resolve(true)), - releaseToGithub: sinon.stub().returns(Promise.resolve(true)) + shouldDoPublicRelease: sinon.stub().resolves(false) + }); + + barque = createStubBarque({ + releaseToBarque: sinon.stub().resolves(true) + }); + + await buildAndUpload( + config, + githubRepo, + barque, + compileAndZipExecutable, + uploadToEvergreen, + uploadToDownloadCenter + ); + + expect(barque.releaseToBarque).to.not.have.been.called; + }); + + it('releases to downloads centre if a public release', async () => { + githubRepo = createStubRepo({ + shouldDoPublicRelease: sinon.stub().resolves(true), + releaseToGithub: sinon.stub().resolves(true) } as any); + barque = createStubBarque({ + releaseToBarque: sinon.stub().resolves(true) + }); + await buildAndUpload( config, githubRepo, + barque, compileAndZipExecutable, uploadToEvergreen, uploadToDownloadCenter @@ -115,4 +200,26 @@ describe('buildAndRelease', () => { expect(githubRepo.releaseToGithub).to.have.been.calledWith(tarballFile, config); }); + + it('does not release to downloads centre if not a public release', async () => { + githubRepo = createStubRepo({ + shouldDoPublicRelease: sinon.stub().resolves(false), + releaseToGithub: sinon.stub().resolves(true) + } as any); + + barque = createStubBarque({ + releaseToBarque: sinon.stub().resolves(true) + }); + + await buildAndUpload( + config, + githubRepo, + barque, + compileAndZipExecutable, + uploadToEvergreen, + uploadToDownloadCenter + ); + + expect(githubRepo.releaseToGithub).to.not.have.been.called; + }); }); diff --git a/packages/build/src/build-and-upload.ts b/packages/build/src/build-and-upload.ts index 3f918fe157..5772e74f18 100644 --- a/packages/build/src/build-and-upload.ts +++ b/packages/build/src/build-and-upload.ts @@ -1,11 +1,14 @@ +import { redactConfig } from './redact-config'; +import { getArtifactUrl } from './evergreen'; import { GithubRepo } from './github-repo'; -import Config from './config'; import { TarballFile } from './tarball'; -import { redactConfig } from './redact-config'; +import { Barque } from './barque'; +import Config from './config'; export default async function buildAndUpload( config: Config, githubRepo: GithubRepo, + barque: Barque, compileAndZipExecutable: (Config) => Promise, uploadToEvergreen: (artifact: string, awsKey: string, awsSecret: string, project: string, revision: string) => Promise, uploadToDownloadCenter: (artifact: string, awsKey: string, awsSecret: string) => Promise): Promise { @@ -30,6 +33,8 @@ export default async function buildAndUpload( ); console.log('mongosh: internal release completed.'); + const evergreenTarball = getArtifactUrl(config.project, config.revision, tarballFile.path); + // Only release to public from master and when tagged with the right version. if (await githubRepo.shouldDoPublicRelease(config)) { console.log('mongosh: start public release.'); @@ -40,6 +45,9 @@ export default async function buildAndUpload( config.downloadCenterAwsSecret ); + await barque.releaseToBarque(evergreenTarball); + console.log('mongosh: submitting to barque complete'); + await githubRepo.releaseToGithub(tarballFile, config); } diff --git a/packages/build/src/evergreen.ts b/packages/build/src/evergreen.ts index f95693e0ff..5f43817a36 100644 --- a/packages/build/src/evergreen.ts +++ b/packages/build/src/evergreen.ts @@ -29,9 +29,15 @@ const uploadArtifactToEvergreen = (artifact: string, awsKey: string, awsSecret: Key: key, Body: fs.createReadStream(artifact) }; + console.log(`mongosh: uploading ${artifact} to evergreen bucket:`, BUCKET, key); console.log(`mongosh: artifact download url: https://s3.amazonaws.com/${BUCKET}/${key}`); return upload(uploadParams, s3); }; +const getArtifactUrl = (project: string, revision: string, artifact: string): string => { + return `https://s3.amazonaws.com/${BUCKET}/${project}/${revision}/${path.basename(artifact)}` +}; + export default uploadArtifactToEvergreen; +export { getArtifactUrl }; diff --git a/packages/build/src/release.ts b/packages/build/src/release.ts index 6a91814dec..674f4d60e3 100644 --- a/packages/build/src/release.ts +++ b/packages/build/src/release.ts @@ -1,13 +1,14 @@ /* eslint-disable no-shadow */ import uploadToDownloadCenter from './upload-to-download-center'; import compileAndZipExecutable from './compile-and-zip-executable'; +import uploadDownloadCenterConfig from './download-center'; import uploadArtifactToEvergreen from './evergreen'; +import buildAndUpload from './build-and-upload'; import { GithubRepo } from './github-repo'; import { Octokit } from '@octokit/rest'; -import Config from './config'; -import buildAndUpload from './build-and-upload'; +import { Barque } from './barque'; import publish from './publish'; -import uploadDownloadCenterConfig from './download-center'; +import Config from './config'; /** * Run the release process. @@ -24,11 +25,13 @@ export default async function release( }); const githubRepo = new GithubRepo(config.repo, octokit); + const barque = new Barque(config); if (command === 'package') { await buildAndUpload( config, githubRepo, + barque, compileAndZipExecutable, uploadArtifactToEvergreen, uploadToDownloadCenter diff --git a/scripts/import-expansions.js b/scripts/import-expansions.js index b18370cc9c..c8328f0710 100644 --- a/scripts/import-expansions.js +++ b/scripts/import-expansions.js @@ -33,4 +33,4 @@ const importExpansions = () => { console.info('Imported expansions:', Object.keys(expansions).join(', ')); } -importExpansions(); \ No newline at end of file +importExpansions();