diff --git a/.evergreen/buildvariants-and-tasks.in.yml b/.evergreen/buildvariants-and-tasks.in.yml index d1a3ae4195b..b7546097dd6 100644 --- a/.evergreen/buildvariants-and-tasks.in.yml +++ b/.evergreen/buildvariants-and-tasks.in.yml @@ -424,6 +424,7 @@ tasks: - func: bootstrap vars: scope: "@mongodb-js/compass-web" + - func: upload-web - func: publish-web - name: publish-dev-release-info diff --git a/.evergreen/buildvariants-and-tasks.yml b/.evergreen/buildvariants-and-tasks.yml index 92f74025242..3f1c2818c69 100644 --- a/.evergreen/buildvariants-and-tasks.yml +++ b/.evergreen/buildvariants-and-tasks.yml @@ -428,6 +428,7 @@ tasks: - func: bootstrap vars: scope: '@mongodb-js/compass-web' + - func: upload-web - func: publish-web - name: publish-dev-release-info tags: [] diff --git a/.evergreen/functions.yml b/.evergreen/functions.yml index 752989a4125..c1f32d78bc7 100644 --- a/.evergreen/functions.yml +++ b/.evergreen/functions.yml @@ -1,5 +1,5 @@ -# The variables are with the functions because they are only used by the -# functions and also because you can't use variables across includes. +# The variables and paratemers are with the functions because they are only used +# by the functions and also because you can't use variables across includes. variables: - &save-artifact-params-private aws_key: ${aws_key} @@ -77,6 +77,13 @@ variables: E2E_TESTS_ATLAS_CS_WITHOUT_SEARCH: ${e2e_tests_atlas_cs_without_search} E2E_TESTS_ATLAS_CS_WITH_SEARCH: ${e2e_tests_atlas_cs_with_search} +parameters: + - key: compass_web_publish_environment + value: dev + description: Atlas Cloud environment that compass-web will be deployed for during a CI run + - key: compass_web_release_commit + description: Exact commit that will be deployed as a latest release for provided compass_web_publish_environment + # This is here with the variables because anchors aren't supported across includes post: - command: archive.targz_pack @@ -521,7 +528,7 @@ functions: echo "Uploading release assets to S3 and GitHub if needed..." npm run --workspace mongodb-compass upload - publish-web: + upload-web: - command: ec2.assume_role params: role_arn: ${downloads_bucket_role_arn} @@ -540,6 +547,29 @@ functions: echo "Uploading release assets to S3" npm run --workspace @mongodb-js/compass-web upload + publish-web: + - command: ec2.assume_role + params: + role_arn: ${downloads_bucket_role_arn} + - command: shell.exec + params: + working_dir: src + shell: bash + env: + <<: *compass-env + COMPASS_WEB_PUBLISH_ENVIRONMENT: '${compass_web_publish_environment}' + COMPASS_WEB_RELEASE_COMMIT: '${compass_web_release_commit}' + script: | + set -e + # Load environment variables + eval $(.evergreen/print-compass-env.sh) + # Deploy to dev on every commit to main, do nothing for release branches + if [[ "$EVERGREEN_PROJECT" == '10gen-compass-main' && "$EVERGREEN_IS_PATCH" != "true" ]]; then + npm run --workspace @mongodb-js/compass-web publish + else + echo "Skipping publish: wrong project ($EVERGREEN_PROJECT) or patch ($EVERGREEN_IS_PATCH)" + fi + publish-dev-release-info: - command: shell.exec params: diff --git a/packages/compass-web/package.json b/packages/compass-web/package.json index 40308874b11..9b7d6ba7f5c 100644 --- a/packages/compass-web/package.json +++ b/packages/compass-web/package.json @@ -63,7 +63,8 @@ "test-watch": "npm run test -- --watch", "test-ci": "npm run test-cov", "reformat": "npm run eslint . -- --fix && npm run prettier -- --write .", - "upload": "node --experimental-strip-types scripts/upload.mts" + "upload": "node --experimental-strip-types scripts/release/upload.mts", + "publish": "node --experimental-strip-types scripts/release/publish.mts" }, "peerDependencies": { "react": "^17.0.2", diff --git a/packages/compass-web/scripts/release/publish.mts b/packages/compass-web/scripts/release/publish.mts new file mode 100644 index 00000000000..7687fb04695 --- /dev/null +++ b/packages/compass-web/scripts/release/publish.mts @@ -0,0 +1,134 @@ +import path from 'path'; +import { brotliCompressSync } from 'zlib'; +import { inspect } from 'util'; +import { + ALLOWED_PUBLISH_ENVIRONMENTS, + DOWNLOADS_BUCKET, + DOWNLOADS_BUCKET_PUBLIC_HOST, + ENTRYPOINT_FILENAME, + MANIFEST_FILENAME, + PUBLISH_ENVIRONMENT, + RELEASE_COMMIT, + asyncPutObject, + getObjectKey, +} from './utils.mts'; + +console.log( + 'Publishing compass-web@%s to %s environment', + RELEASE_COMMIT, + PUBLISH_ENVIRONMENT +); + +if (!ALLOWED_PUBLISH_ENVIRONMENTS.includes(PUBLISH_ENVIRONMENT ?? '')) { + throw new Error( + `Unknown publish environment: expected ${inspect( + ALLOWED_PUBLISH_ENVIRONMENTS + )}, got ${inspect(PUBLISH_ENVIRONMENT)}` + ); +} + +const publicManifestUrl = new URL( + getObjectKey(MANIFEST_FILENAME), + DOWNLOADS_BUCKET_PUBLIC_HOST +); + +const publicEntrypointUrl = new URL( + getObjectKey(ENTRYPOINT_FILENAME), + DOWNLOADS_BUCKET_PUBLIC_HOST +); + +let assets: URL[]; + +function assertResponseIsOk(res: Response) { + if (res.status !== 200) { + throw new Error( + `Request returned a non-OK response: ${res.status} ${res.statusText}` + ); + } +} + +console.log('Fetching manifest from %s', publicManifestUrl); + +try { + const res = await fetch(publicManifestUrl); + assertResponseIsOk(res); + const manifest = await res.json(); + + if ( + !( + Array.isArray(manifest) && + manifest.every((asset) => { + return typeof asset === 'string'; + }) + ) + ) { + throw new Error( + `Manifest schema is not matching: expected string[], got ${inspect( + manifest + )}` + ); + } + + console.log('Checking that assets in manifest exist'); + + assets = manifest.map((asset) => { + return new URL(getObjectKey(asset), DOWNLOADS_BUCKET_PUBLIC_HOST); + }); + + await Promise.all( + assets.map(async (assetUrl) => { + const res = await fetch(assetUrl, { method: 'HEAD' }); + assertResponseIsOk(res); + }) + ); +} catch (err) { + throw new AggregateError( + [err], + `Aborting publish, failed to resolve manifest ${publicManifestUrl}` + ); +} + +function buildProxyEntrypointFile() { + return ( + assets + .map((asset) => { + return `import ${JSON.stringify(asset)};`; + }) + .concat( + `export * from ${JSON.stringify(publicEntrypointUrl)};`, + `/** Compass version: https://github.com/mongodb-js/compass/tree/${RELEASE_COMMIT} */` + ) + .join('\n') + '\n' + ); +} + +const fileKey = getObjectKey('index.mjs', PUBLISH_ENVIRONMENT); +const fileContent = buildProxyEntrypointFile(); +const compressedFileContent = brotliCompressSync(fileContent); + +console.log( + 'Uploading entrypoint to s3://%s/%s ...', + DOWNLOADS_BUCKET, + fileKey +); + +const ENTRYPOINT_CACHE_MAX_AGE_SECONDS = 1 * 60 * 3; // 3mins + +const res = await asyncPutObject({ + ACL: 'private', + Bucket: DOWNLOADS_BUCKET, + Key: fileKey, + Body: compressedFileContent, + ContentType: 'text/javascript', + ContentEncoding: 'br', + ContentLength: compressedFileContent.byteLength, + // "Latest" entrypoint files can change quite often, so max-age is quite + // short and browser should always revalidate on stale + CacheControl: `public, max-age=${ENTRYPOINT_CACHE_MAX_AGE_SECONDS}, must-revalidate`, +}); + +console.log( + 'Successfully uploaded %s (ETag: %s)', + path.basename(fileKey), + res.ETag +); diff --git a/packages/compass-web/scripts/release/upload.mts b/packages/compass-web/scripts/release/upload.mts new file mode 100644 index 00000000000..03e7059ffa6 --- /dev/null +++ b/packages/compass-web/scripts/release/upload.mts @@ -0,0 +1,62 @@ +import fs from 'fs'; +import path from 'path'; +import { brotliCompressSync } from 'zlib'; +import { + DIST_DIR, + DOWNLOADS_BUCKET, + asyncPutObject, + getObjectKey, +} from './utils.mts'; + +const artifacts = await fs.promises.readdir(DIST_DIR); + +if (!artifacts.length) { + throw new Error('No artifact files found'); +} + +const contentTypeForExt: Record = { + '.mjs': 'text/javascript', + '.txt': 'text/plain', // extracted third party license info + '.ts': 'text/typescript', // type definitions + '.json': 'application/json', // tsdoc / assets meta +}; + +const ALLOWED_EXTS = Object.keys(contentTypeForExt); + +for (const file of artifacts) { + if (!ALLOWED_EXTS.includes(path.extname(file))) { + throw new Error(`Unexpected artifact file extension for ${file}`); + } +} + +const IMMUTABLE_CACHE_MAX_AGE_SECONDS = 1 * 60 * 60 * 24 * 7; // a week + +for (const file of artifacts) { + const filePath = path.join(DIST_DIR, file); + const objectKey = getObjectKey(file); + + console.log( + 'Uploading compass-web/dist/%s to s3://%s/%s ...', + file, + DOWNLOADS_BUCKET, + objectKey + ); + + const fileContent = fs.readFileSync(filePath, 'utf8'); + const compressedFileContent = brotliCompressSync(fileContent); + + const res = await asyncPutObject({ + ACL: 'private', + Bucket: DOWNLOADS_BUCKET, + Key: objectKey, + Body: compressedFileContent, + ContentType: contentTypeForExt[path.extname(file)], + ContentEncoding: 'br', + ContentLength: compressedFileContent.byteLength, + // Assets stored under the commit hash never change after upload, so the + // cache-control setting for them can be quite generous + CacheControl: `public, max-age=${IMMUTABLE_CACHE_MAX_AGE_SECONDS}, immutable`, + }); + + console.log('Successfully uploaded %s (ETag: %s)', file, res.ETag); +} diff --git a/packages/compass-web/scripts/release/utils.mts b/packages/compass-web/scripts/release/utils.mts new file mode 100644 index 00000000000..76c03bf82e1 --- /dev/null +++ b/packages/compass-web/scripts/release/utils.mts @@ -0,0 +1,55 @@ +import S3 from 'aws-sdk/clients/s3.js'; +import child_process from 'child_process'; +import path from 'path'; +import { promisify } from 'util'; + +// TODO(SRE-4971): replace with a compass-web-only bucket when provisioned +export const DOWNLOADS_BUCKET = 'cdn-origin-compass'; + +export const DOWNLOADS_BUCKET_PUBLIC_HOST = 'https://downloads.mongodb.com'; + +export const ENTRYPOINT_FILENAME = 'compass-web.mjs'; + +export const MANIFEST_FILENAME = 'assets-manifest.json'; + +export const DIST_DIR = path.resolve(import.meta.dirname, '..', '..', 'dist'); + +export const ALLOWED_PUBLISH_ENVIRONMENTS = ['dev', 'qa', 'staging', 'prod']; + +export const PUBLISH_ENVIRONMENT = process.env.COMPASS_WEB_PUBLISH_ENVIRONMENT; + +export const RELEASE_COMMIT = + process.env.COMPASS_WEB_RELEASE_COMMIT || + child_process + .spawnSync('git', ['rev-parse', 'HEAD'], { encoding: 'utf8' }) + .stdout.trim(); + +function getAWSCredentials() { + if ( + !process.env.DOWNLOAD_CENTER_NEW_AWS_ACCESS_KEY_ID || + !process.env.DOWNLOAD_CENTER_NEW_AWS_SECRET_ACCESS_KEY || + !process.env.DOWNLOAD_CENTER_NEW_AWS_SESSION_TOKEN + ) { + throw new Error('Missing required env variables'); + } + return { + accessKeyId: process.env.DOWNLOAD_CENTER_NEW_AWS_ACCESS_KEY_ID, + secretAccessKey: process.env.DOWNLOAD_CENTER_NEW_AWS_SECRET_ACCESS_KEY, + sessionToken: process.env.DOWNLOAD_CENTER_NEW_AWS_SESSION_TOKEN, + }; +} +const s3Client = new S3({ + credentials: getAWSCredentials(), +}); + +export const asyncPutObject: ( + params: S3.Types.PutObjectRequest +) => Promise = promisify( + s3Client.putObject.bind(s3Client) +); + +export function getObjectKey(filename: string, release = RELEASE_COMMIT) { + // TODO(SRE-4971): while we're uploading to the downloads bucket, the object + // key always needs to start with `compass/` + return `compass/compass-web/${release}/${filename}`; +} diff --git a/packages/compass-web/scripts/upload.mts b/packages/compass-web/scripts/upload.mts deleted file mode 100644 index a6b84fd47db..00000000000 --- a/packages/compass-web/scripts/upload.mts +++ /dev/null @@ -1,95 +0,0 @@ -import fs from 'fs'; -import path from 'path'; -import child_process from 'child_process'; -import { brotliCompressSync } from 'zlib'; -import { promisify } from 'util'; -import S3 from 'aws-sdk/clients/s3.js'; - -// TODO(SRE-4971): replace with a compass-web-only bucket when provisioned -const DOWNLOADS_BUCKET = 'cdn-origin-compass'; - -const DIST_DIR = path.resolve(import.meta.dirname, '..', 'dist'); - -const HEAD = child_process - .spawnSync('git', ['rev-parse', 'HEAD'], { encoding: 'utf8' }) - .stdout.trim(); - -function getCredentials() { - if ( - !process.env.DOWNLOAD_CENTER_NEW_AWS_ACCESS_KEY_ID || - !process.env.DOWNLOAD_CENTER_NEW_AWS_SECRET_ACCESS_KEY || - !process.env.DOWNLOAD_CENTER_NEW_AWS_SESSION_TOKEN - ) { - throw new Error('Missing required env variables'); - } - return { - accessKeyId: process.env.DOWNLOAD_CENTER_NEW_AWS_ACCESS_KEY_ID, - secretAccessKey: process.env.DOWNLOAD_CENTER_NEW_AWS_SECRET_ACCESS_KEY, - sessionToken: process.env.DOWNLOAD_CENTER_NEW_AWS_SESSION_TOKEN, - }; -} - -const artifacts = await fs.promises.readdir(DIST_DIR); - -if (!artifacts.length) { - throw new Error('No artifact files found'); -} - -const contentTypeForExt: Record = { - '.mjs': 'text/javascript', - '.txt': 'text/plain', // extracted third party license info - '.ts': 'text/typescript', // type definitions - '.json': 'application/json', // tsdoc meta -}; - -const ALLOWED_EXTS = Object.keys(contentTypeForExt); - -for (const file of artifacts) { - if (!ALLOWED_EXTS.includes(path.extname(file))) { - throw new Error(`Unexpected artifact file extension for ${file}`); - } -} - -const s3Client = new S3({ - credentials: getCredentials(), -}); - -const IMMUTABLE_CACHE_MAX_AGE = 1000 * 60 * 60 * 24 * 7; // a week - -for (const file of artifacts) { - const filePath = path.join(DIST_DIR, file); - // TODO(SRE-4971): while we're uploading to the downloads bucket, the object - // key always needs to start with `compass/` - const objectKey = `compass/web/${HEAD}/${file}`; - - console.log( - 'Uploading compass-web/dist/%s to s3://%s/%s ...', - file, - DOWNLOADS_BUCKET, - objectKey - ); - - const fileContent = fs.readFileSync(filePath, 'utf8'); - const compressedFileContent = brotliCompressSync(fileContent); - - const asyncPutObject: ( - params: S3.Types.PutObjectRequest - ) => Promise = promisify( - s3Client.putObject.bind(s3Client) - ); - - const res = await asyncPutObject({ - ACL: 'private', - Bucket: DOWNLOADS_BUCKET, - Key: objectKey, - Body: compressedFileContent, - ContentType: contentTypeForExt[path.extname(file)], - ContentEncoding: 'br', - ContentLength: compressedFileContent.byteLength, - // Assets stored under the commit hash never change after upload, so the - // cache-control setting for them can be quite generous - CacheControl: `public, max-age=${IMMUTABLE_CACHE_MAX_AGE}, immutable`, - }); - - console.log('Successfully uploaded %s (ETag: %s)', file, res.ETag); -} diff --git a/packages/compass-web/tsconfig-build.json b/packages/compass-web/tsconfig-build.json index 737091e2e1c..c14aec029a8 100644 --- a/packages/compass-web/tsconfig-build.json +++ b/packages/compass-web/tsconfig-build.json @@ -1,5 +1,10 @@ { "extends": "./tsconfig.json", + "compilerOptions": { + // reversing options that are not relevant for the build + "allowImportingTsExtensions": false, + "noEmit": false + }, "include": ["src/**/*"], "exclude": ["./src/**/*.spec.*"] } diff --git a/packages/compass-web/tsconfig.json b/packages/compass-web/tsconfig.json index f27cc34b276..4e12ca2f534 100644 --- a/packages/compass-web/tsconfig.json +++ b/packages/compass-web/tsconfig.json @@ -1,7 +1,10 @@ { "extends": "@mongodb-js/tsconfig-compass/tsconfig.react.json", "compilerOptions": { - "outDir": "dist" + "outDir": "dist", + // for scripts that we run with Node.js strip-types option + "allowImportingTsExtensions": true, + "noEmit": true }, "include": ["**/*"], "exclude": ["node_modules", "dist", "test/types"]