From 782b4b30af02af529f8eccccd9809f5c99939758 Mon Sep 17 00:00:00 2001 From: Dylan Richardson Date: Tue, 25 Mar 2025 16:30:52 +0000 Subject: [PATCH] COMPASS-9160: dual write release artifacts to new bucket This commit adjusts Compass to write release artifacts to a new S3 bucket. Eventually the old bucket will be removed after we have confirmed that we can serve artifacts from the new bucket. We also use assume role to access this bucket instead of static credentials. --- .evergreen/functions.yml | 6 +++++ package-lock.json | 16 +++++++------- packages/hadron-build/commands/upload.js | 2 ++ packages/hadron-build/lib/download-center.js | 23 ++++++++++++++++++++ packages/hadron-build/package.json | 2 +- 5 files changed, 40 insertions(+), 9 deletions(-) diff --git a/.evergreen/functions.yml b/.evergreen/functions.yml index b1023484cd2..c25d59d0cc8 100644 --- a/.evergreen/functions.yml +++ b/.evergreen/functions.yml @@ -504,12 +504,18 @@ functions: content_type: application/x-gzip publish: + - command: ec2.assume_role + params: + role_arn: "arn:aws:iam::119629040606:role/s3-access.cdn-origin-compass" - command: shell.exec params: working_dir: src shell: bash env: <<: *compass-env + DOWNLOAD_CENTER_NEW_AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID} + DOWNLOAD_CENTER_NEW_AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY} + DOWNLOAD_CENTER_NEW_AWS_SESSION_TOKEN: ${AWS_SESSION_TOKEN} script: | set -e # Load environment variables diff --git a/package-lock.json b/package-lock.json index 04b44b8bcad..2cc4bdcfa10 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8116,9 +8116,9 @@ } }, "node_modules/@mongodb-js/dl-center": { - "version": "1.1.7", - "resolved": "https://registry.npmjs.org/@mongodb-js/dl-center/-/dl-center-1.1.7.tgz", - "integrity": "sha512-UEEQpiGAX5v5VMFsfUTG94fI/f6kCU03MgU4YTvn1JF2E8nM0kXrcPbCc8FOhqGVTlAs8+tYwv9lLp7hbJlXqQ==", + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@mongodb-js/dl-center/-/dl-center-1.2.0.tgz", + "integrity": "sha512-dieKzE+VmROm+IPij7Pv1nSBt4UP/KlKSQYw2LXq9ST5hFIrQ1A0eWSmjl3I4u9CE7y5reBwiy0sBFvOXnn8TA==", "license": "Apache-2.0", "dependencies": { "ajv": "^6.12.5", @@ -48712,7 +48712,7 @@ "dependencies": { "@electron/rebuild": "^3.7.1", "@mongodb-js/devtools-github-repo": "^1.4.1", - "@mongodb-js/dl-center": "^1.1.7", + "@mongodb-js/dl-center": "^1.2.0", "@mongodb-js/electron-wix-msi": "^3.0.0", "@mongodb-js/signing-utils": "^0.3.8", "@npmcli/arborist": "^6.2.0", @@ -59637,9 +59637,9 @@ } }, "@mongodb-js/dl-center": { - "version": "1.1.7", - "resolved": "https://registry.npmjs.org/@mongodb-js/dl-center/-/dl-center-1.1.7.tgz", - "integrity": "sha512-UEEQpiGAX5v5VMFsfUTG94fI/f6kCU03MgU4YTvn1JF2E8nM0kXrcPbCc8FOhqGVTlAs8+tYwv9lLp7hbJlXqQ==", + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@mongodb-js/dl-center/-/dl-center-1.2.0.tgz", + "integrity": "sha512-dieKzE+VmROm+IPij7Pv1nSBt4UP/KlKSQYw2LXq9ST5hFIrQ1A0eWSmjl3I4u9CE7y5reBwiy0sBFvOXnn8TA==", "requires": { "ajv": "^6.12.5", "aws-sdk": "^2.1441.0", @@ -74106,7 +74106,7 @@ "requires": { "@electron/rebuild": "^3.7.1", "@mongodb-js/devtools-github-repo": "^1.4.1", - "@mongodb-js/dl-center": "^1.1.7", + "@mongodb-js/dl-center": "^1.2.0", "@mongodb-js/electron-wix-msi": "^3.0.0", "@mongodb-js/eslint-config-compass": "^1.3.5", "@mongodb-js/signing-utils": "^0.3.8", diff --git a/packages/hadron-build/commands/upload.js b/packages/hadron-build/commands/upload.js index 340847b6637..ef060bbbff7 100644 --- a/packages/hadron-build/commands/upload.js +++ b/packages/hadron-build/commands/upload.js @@ -19,6 +19,7 @@ const { getKeyPrefix, downloadManifest, uploadAsset, + uploadAssetNew, uploadManifest, } = require('../lib/download-center'); @@ -235,6 +236,7 @@ async function uploadAssetsToDownloadCenter(assets, channel, dryRun) { ); if (!dryRun) { await uploadAsset(channel, asset); + await uploadAssetNew(channel, asset); } cli.info(`${asset.name}: upload to download center completed.`); }); diff --git a/packages/hadron-build/lib/download-center.js b/packages/hadron-build/lib/download-center.js index b68ed07b3d1..fd5a8eb2f6e 100644 --- a/packages/hadron-build/lib/download-center.js +++ b/packages/hadron-build/lib/download-center.js @@ -6,6 +6,7 @@ const { DownloadCenter } = require('@mongodb-js/dl-center'); const download = require('download'); const DOWNLOADS_BUCKET = 'downloads.10gen.com'; +const DOWNLOADS_BUCKET_NEW = 'cdn-origin-compass'; const MANIFEST_BUCKET = 'info-mongodb-com'; const MANIFEST_OBJECT_KEY = 'com-download-center/compass.json'; @@ -32,10 +33,31 @@ const getDownloadCenter = (bucketConfig) => { }); }; +const getDownloadCenterNew = (bucketConfig) => { + requireEnvironmentVariables([ + 'DOWNLOAD_CENTER_NEW_AWS_ACCESS_KEY_ID', + 'DOWNLOAD_CENTER_NEW_AWS_SECRET_ACCESS_KEY', + 'DOWNLOAD_CENTER_NEW_AWS_SESSION_TOKEN', + ]); + + return new DownloadCenter({ + ...bucketConfig, + accessKeyId: process.env.DOWNLOAD_CENTER_NEW_AWS_ACCESS_KEY_ID, + secretAccessKey: process.env.DOWNLOAD_CENTER_NEW_AWS_SECRET_ACCESS_KEY, + sessionToken: process.env.DOWNLOAD_CENTER_NEW_AWS_SESSION_TOKEN, + }); +}; + const getKeyPrefix = (channel) => { return channel && channel !== 'stable' ? `compass/${channel}` : 'compass'; }; +const uploadAssetNew = async (channel, asset) => { + const dlCenterNew = getDownloadCenterNew({ bucket: DOWNLOADS_BUCKET_NEW }); + const objectKey = `${getKeyPrefix(channel)}/${asset.name}`; + return dlCenterNew.uploadAsset(objectKey, fs.createReadStream(asset.path)); +}; + const uploadAsset = async (channel, asset) => { const dlCenter = getDownloadCenter({ bucket: DOWNLOADS_BUCKET }); const objectKey = `${getKeyPrefix(channel)}/${asset.name}`; @@ -75,6 +97,7 @@ module.exports = { getDownloadCenter, getKeyPrefix, uploadAsset, + uploadAssetNew, downloadManifest, uploadManifest, downloadAssetFromEvergreen, diff --git a/packages/hadron-build/package.json b/packages/hadron-build/package.json index 9de168f1421..4bb13ccd1ef 100644 --- a/packages/hadron-build/package.json +++ b/packages/hadron-build/package.json @@ -21,7 +21,7 @@ "dependencies": { "@electron/rebuild": "^3.7.1", "@mongodb-js/devtools-github-repo": "^1.4.1", - "@mongodb-js/dl-center": "^1.1.7", + "@mongodb-js/dl-center": "^1.2.0", "@mongodb-js/electron-wix-msi": "^3.0.0", "@mongodb-js/signing-utils": "^0.3.8", "@npmcli/arborist": "^6.2.0",