feat(web, ci): add publish script; automatically publish for dev on every merge to main COMPASS-10156

.evergreen/buildvariants-and-tasks.in.yml

@@ -424,6 +424,7 @@ tasks:
       - func: bootstrap
         vars:
           scope: "@mongodb-js/compass-web"
+      - func: upload-web
       - func: publish-web
 
   - name: publish-dev-release-info

.evergreen/buildvariants-and-tasks.yml

@@ -428,6 +428,7 @@ tasks:
       - func: bootstrap
         vars:
           scope: '@mongodb-js/compass-web'
+      - func: upload-web
       - func: publish-web
   - name: publish-dev-release-info
     tags: []

.evergreen/functions.yml

@@ -1,5 +1,5 @@
-# The variables are with the functions because they are only used by the
-# functions and also because you can't use variables across includes.
+# The variables and paratemers are with the functions because they are only used
+# by the functions and also because you can't use variables across includes.
 variables:
   - &save-artifact-params-private
     aws_key: ${aws_key}
@@ -77,6 +77,13 @@ variables:
     E2E_TESTS_ATLAS_CS_WITHOUT_SEARCH: ${e2e_tests_atlas_cs_without_search}
     E2E_TESTS_ATLAS_CS_WITH_SEARCH: ${e2e_tests_atlas_cs_with_search}
 
+parameters:
+  - key: compass_web_publish_environment
+    value: dev
+    description: Atlas Cloud environment that compass-web will be deployed for during a CI run
+  - key: compass_web_release_commit
+    description: Exact commit that will be deployed as a latest release for provided compass_web_publish_environment
+
 # This is here with the variables because anchors aren't supported across includes
 post:
   - command: archive.targz_pack
@@ -521,7 +528,7 @@ functions:
           echo "Uploading release assets to S3 and GitHub if needed..."
           npm run --workspace mongodb-compass upload
 
-  publish-web:
+  upload-web:
     - command: ec2.assume_role
       params:
         role_arn: ${downloads_bucket_role_arn}
@@ -540,6 +547,29 @@ functions:
           echo "Uploading release assets to S3"
           npm run --workspace @mongodb-js/compass-web upload
 
+  publish-web:
+    - command: ec2.assume_role
+      params:
+        role_arn: ${downloads_bucket_role_arn}
+    - command: shell.exec
+      params:
+        working_dir: src
+        shell: bash
+        env:
+          <<: *compass-env
+          COMPASS_WEB_PUBLISH_ENVIRONMENT: '${compass_web_publish_environment}'
+          COMPASS_WEB_RELEASE_COMMIT: '${compass_web_release_commit}'
+        script: |
+          set -e
+          # Load environment variables
+          eval $(.evergreen/print-compass-env.sh)
+          # Deploy to dev on every commit to main, do nothing for release branches
+          if [[ "$EVERGREEN_PROJECT" == '10gen-compass-main' && "$EVERGREEN_IS_PATCH" != "true" ]]; then
+            npm run --workspace @mongodb-js/compass-web publish
+          else 
+            echo "Skipping publish: wrong project ($EVERGREEN_PROJECT) or patch ($EVERGREEN_IS_PATCH)"
+          fi
+
   publish-dev-release-info:
     - command: shell.exec
       params:

packages/compass-web/package.json

@@ -63,7 +63,8 @@
     "test-watch": "npm run test -- --watch",
     "test-ci": "npm run test-cov",
     "reformat": "npm run eslint . -- --fix && npm run prettier -- --write .",
-    "upload": "node --experimental-strip-types scripts/upload.mts"
+    "upload": "node --experimental-strip-types scripts/release/upload.mts",
+    "publish": "node --experimental-strip-types scripts/release/publish.mts"
   },
   "peerDependencies": {
     "react": "^17.0.2",

packages/compass-web/scripts/release/publish.mts

@@ -0,0 +1,134 @@
+import path from 'path';
+import { brotliCompressSync } from 'zlib';
+import { inspect } from 'util';
+import {
+  ALLOWED_PUBLISH_ENVIRONMENTS,
+  DOWNLOADS_BUCKET,
+  DOWNLOADS_BUCKET_PUBLIC_HOST,
+  ENTRYPOINT_FILENAME,
+  MANIFEST_FILENAME,
+  PUBLISH_ENVIRONMENT,
+  RELEASE_COMMIT,
+  asyncPutObject,
+  getObjectKey,
+} from './utils.mts';
+
+console.log(
+  'Publishing compass-web@%s to %s environment',
+  RELEASE_COMMIT,
+  PUBLISH_ENVIRONMENT
+);
+
+if (!ALLOWED_PUBLISH_ENVIRONMENTS.includes(PUBLISH_ENVIRONMENT ?? '')) {
+  throw new Error(
+    `Unknown publish environment: expected ${inspect(
+      ALLOWED_PUBLISH_ENVIRONMENTS
+    )}, got ${inspect(PUBLISH_ENVIRONMENT)}`
+  );
+}
+
+const publicManifestUrl = new URL(
+  getObjectKey(MANIFEST_FILENAME),
+  DOWNLOADS_BUCKET_PUBLIC_HOST
+);
+
+const publicEntrypointUrl = new URL(
+  getObjectKey(ENTRYPOINT_FILENAME),
+  DOWNLOADS_BUCKET_PUBLIC_HOST
+);
+
+let assets: URL[];
+
+function assertResponseIsOk(res: Response) {
+  if (res.status !== 200) {
+    throw new Error(
+      `Request returned a non-OK response: ${res.status} ${res.statusText}`
+    );
+  }
+}
+
+console.log('Fetching manifest from %s', publicManifestUrl);
+
+try {
+  const res = await fetch(publicManifestUrl);
+  assertResponseIsOk(res);
+  const manifest = await res.json();
+
+  if (
+    !(
+      Array.isArray(manifest) &&
+      manifest.every((asset) => {
+        return typeof asset === 'string';
+      })
+    )
+  ) {
+    throw new Error(
+      `Manifest schema is not matching: expected string[], got ${inspect(
+        manifest
+      )}`
+    );
+  }
+
+  console.log('Checking that assets in manifest exist');
+
+  assets = manifest.map((asset) => {
+    return new URL(getObjectKey(asset), DOWNLOADS_BUCKET_PUBLIC_HOST);
+  });
+
+  await Promise.all(
+    assets.map(async (assetUrl) => {
+      const res = await fetch(assetUrl, { method: 'HEAD' });
+      assertResponseIsOk(res);
+    })
+  );
+} catch (err) {
+  throw new AggregateError(
+    [err],
+    `Aborting publish, failed to resolve manifest ${publicManifestUrl}`
+  );
+}
+
+function buildProxyEntrypointFile() {
+  return (
+    assets
+      .map((asset) => {
+        return `import ${JSON.stringify(asset)};`;
+      })
+      .concat(
+        `export * from ${JSON.stringify(publicEntrypointUrl)};`,
+        `/** Compass version: https://github.com/mongodb-js/compass/tree/${RELEASE_COMMIT} */`
+      )
+      .join('\n') + '\n'
+  );
+}
+
+const fileKey = getObjectKey('index.mjs', PUBLISH_ENVIRONMENT);
+const fileContent = buildProxyEntrypointFile();
+const compressedFileContent = brotliCompressSync(fileContent);
+
+console.log(
+  'Uploading entrypoint to s3://%s/%s ...',
+  DOWNLOADS_BUCKET,
+  fileKey
+);
+
+const ENTRYPOINT_CACHE_MAX_AGE_SECONDS = 1 * 60 * 3; // 3mins
+
+const res = await asyncPutObject({
+  ACL: 'private',
+  Bucket: DOWNLOADS_BUCKET,
+  Key: fileKey,
+  Body: compressedFileContent,
+  ContentType: 'text/javascript',
+  ContentEncoding: 'br',
+  ContentLength: compressedFileContent.byteLength,
+  // "Latest" entrypoint files can change quite often, so max-age is quite
+  // short and browser should always revalidate on stale
+  CacheControl: `public, max-age=${ENTRYPOINT_CACHE_MAX_AGE_SECONDS}, must-revalidate`,
+});
+
+console.log(
+  'Successfully uploaded %s (ETag: %s)',
+  path.basename(fileKey),
+  res.ETag
+);

packages/compass-web/scripts/release/upload.mts

@@ -0,0 +1,62 @@
+import fs from 'fs';
+import path from 'path';
+import { brotliCompressSync } from 'zlib';
+import {
+  DIST_DIR,
+  DOWNLOADS_BUCKET,
+  asyncPutObject,
+  getObjectKey,
+} from './utils.mts';
+
+const artifacts = await fs.promises.readdir(DIST_DIR);
+
+if (!artifacts.length) {
+  throw new Error('No artifact files found');
+}
+
+const contentTypeForExt: Record<string, string> = {
+  '.mjs': 'text/javascript',
+  '.txt': 'text/plain', // extracted third party license info
+  '.ts': 'text/typescript', // type definitions
+  '.json': 'application/json', // tsdoc / assets meta
+};
+
+const ALLOWED_EXTS = Object.keys(contentTypeForExt);
+
+for (const file of artifacts) {
+  if (!ALLOWED_EXTS.includes(path.extname(file))) {
+    throw new Error(`Unexpected artifact file extension for ${file}`);
+  }
+}
+
+const IMMUTABLE_CACHE_MAX_AGE_SECONDS = 1 * 60 * 60 * 24 * 7; // a week
+
+for (const file of artifacts) {
+  const filePath = path.join(DIST_DIR, file);
+  const objectKey = getObjectKey(file);
+
+  console.log(
+    'Uploading compass-web/dist/%s to s3://%s/%s ...',
+    file,
+    DOWNLOADS_BUCKET,
+    objectKey
+  );
+
+  const fileContent = fs.readFileSync(filePath, 'utf8');
+  const compressedFileContent = brotliCompressSync(fileContent);
+
+  const res = await asyncPutObject({
+    ACL: 'private',
+    Bucket: DOWNLOADS_BUCKET,
+    Key: objectKey,
+    Body: compressedFileContent,
+    ContentType: contentTypeForExt[path.extname(file)],
+    ContentEncoding: 'br',
+    ContentLength: compressedFileContent.byteLength,
+    // Assets stored under the commit hash never change after upload, so the
+    // cache-control setting for them can be quite generous
+    CacheControl: `public, max-age=${IMMUTABLE_CACHE_MAX_AGE_SECONDS}, immutable`,
+  });
+
+  console.log('Successfully uploaded %s (ETag: %s)', file, res.ETag);
+}

packages/compass-web/scripts/release/utils.mts

@@ -0,0 +1,55 @@
+import S3 from 'aws-sdk/clients/s3.js';
+import child_process from 'child_process';
+import path from 'path';
+import { promisify } from 'util';
+
+// TODO(SRE-4971): replace with a compass-web-only bucket when provisioned
+export const DOWNLOADS_BUCKET = 'cdn-origin-compass';
+
+export const DOWNLOADS_BUCKET_PUBLIC_HOST = 'https://downloads.mongodb.com';
+
+export const ENTRYPOINT_FILENAME = 'compass-web.mjs';
+
+export const MANIFEST_FILENAME = 'assets-manifest.json';
+
+export const DIST_DIR = path.resolve(import.meta.dirname, '..', '..', 'dist');
+
+export const ALLOWED_PUBLISH_ENVIRONMENTS = ['dev', 'qa', 'staging', 'prod'];
+
+export const PUBLISH_ENVIRONMENT = process.env.COMPASS_WEB_PUBLISH_ENVIRONMENT;
+
+export const RELEASE_COMMIT =
+  process.env.COMPASS_WEB_RELEASE_COMMIT ||
+  child_process
+    .spawnSync('git', ['rev-parse', 'HEAD'], { encoding: 'utf8' })
+    .stdout.trim();
+
+function getAWSCredentials() {
+  if (
+    !process.env.DOWNLOAD_CENTER_NEW_AWS_ACCESS_KEY_ID ||
+    !process.env.DOWNLOAD_CENTER_NEW_AWS_SECRET_ACCESS_KEY ||
+    !process.env.DOWNLOAD_CENTER_NEW_AWS_SESSION_TOKEN
+  ) {
+    throw new Error('Missing required env variables');
+  }
+  return {
+    accessKeyId: process.env.DOWNLOAD_CENTER_NEW_AWS_ACCESS_KEY_ID,
+    secretAccessKey: process.env.DOWNLOAD_CENTER_NEW_AWS_SECRET_ACCESS_KEY,
+    sessionToken: process.env.DOWNLOAD_CENTER_NEW_AWS_SESSION_TOKEN,
+  };
+}
+const s3Client = new S3({
+  credentials: getAWSCredentials(),
+});
+
+export const asyncPutObject: (
+  params: S3.Types.PutObjectRequest
+) => Promise<S3.Types.PutObjectOutput> = promisify(
+  s3Client.putObject.bind(s3Client)
+);
+
+export function getObjectKey(filename: string, release = RELEASE_COMMIT) {
+  // TODO(SRE-4971): while we're uploading to the downloads bucket, the object
+  // key always needs to start with `compass/`
+  return `compass/compass-web/${release}/${filename}`;
+}