🏗 release: Update self-hosting support to use amp release (#36165)

* Update self-host to use amp release `amp release` copies static files and downloaded resources to supplement an `amp dist` runtime. Support custom release flavor definitions and update the amp-framework-hosting documentation. * Fix typo in documentation * Prefer accessing argv instead of passing value * Skip cleaning custom configs by default * Review suggestions
ampproject · Oct 1, 2021 · db1659d · db1659d
1 parent e484f7f
commit db1659d
Show file tree

Hide file tree

Showing 6 changed files with 165 additions and 36 deletions.
diff --git a/.gitignore b/.gitignore
@@ -14,6 +14,7 @@ build-system/server/new-server/transforms/dist
 build-system/tasks/performance/cache
 build-system/tasks/performance/results.json
 build-system/global-configs/custom-config.json
+build-system/global-configs/custom-flavors-config.json
 dist
 dist.3p
 dist.tools

diff --git a/build-system/global-configs/README.md b/build-system/global-configs/README.md
@@ -48,6 +48,41 @@ The resulting config is
 }
 ```
 
+# custom-flavors-config.json
+
+Additional release flavors can be defined in `build-system/global-configs/custom-flavors-config.json` and they will automatically be made available to `amp release`. This file should be an array of `DistFlavorDef` objects (see definition in [build-system/tasks/release/index.js](../tasks/release/index.js)). For example:
+
+```json
+[
+  {
+    "flavorType": "custom-exp",
+    "name": "Custom Experimental Release",
+    "environment": "AMP",
+    "rtvPrefixes": [ "00" ],
+    "command": "amp dist --noconfig"
+  },
+  {
+    "flavorType": "custom-prod",
+    "name": "Custom Production Release",
+    "environment": "AMP",
+    "rtvPrefixes": [ "01" ],
+    "command": "amp dist --noconfig"
+  }
+]
+```
+
+and then "Custom Production Release" could be built with:
+
+```sh
+amp release --flavor="custom-prod"
+```
+
+**Tips:**
+
+-   Be sure to pass flag `--noconfig` to `amp dist` in the flavor command, otherwise you will end up with multiple `AMP_CONFIG` definitions in entrypoint files (`v0.js`, `shadow-v0.js`, etc.).
+-   Flag `--version_override` is not supported.
+-   `AMP_CONFIG` can be customized with [`custom-config.json`](#custom-configjson) to further tailor the build.
+
 # client-side-experiments-config.json
 
 This config is used to run client side diverted experiments, adding runtime support for deploying AMP experiment configuration updates faster via the CDN and cache pages. It is complimentary to `{canary,prod,custom}-config.json` and takes precedence over them. (See I2I issue: [#34013](https://github.com/ampproject/amphtml/issues/34013))

diff --git a/build-system/tasks/clean.js b/build-system/tasks/clean.js
@@ -2,8 +2,9 @@
 
 const argv = require('minimist')(process.argv.slice(2));
 const del = require('del');
+const fs = require('fs-extra');
 const path = require('path');
-const {cyan} = require('kleur/colors');
+const {cyan, yellow} = require('kleur/colors');
 const {log} = require('../common/logging');
 
 const ROOT_DIR = path.resolve(__dirname, '../../');
@@ -30,7 +31,6 @@ async function clean() {
     'build-system/server/new-server/transforms/dist',
     'build-system/tasks/performance/cache',
     'build-system/tasks/performance/results.json',
-    'build-system/global-configs/custom-config.json',
     'dist',
     'dist.3p',
     'dist.tools',
@@ -47,6 +47,19 @@ async function clean() {
   if (argv.include_subpackages) {
     pathsToDelete.push('**/node_modules', '!node_modules');
   }
+  const customConfigs = [
+    'build-system/global-configs/custom-config.json',
+    'build-system/global-configs/custom-flavors-config.json',
+  ];
+  if (argv.include_custom_configs) {
+    pathsToDelete.push(...customConfigs);
+  } else {
+    for (const customConfig of customConfigs) {
+      if (fs.existsSync(customConfig)) {
+        log(yellow('Skipping path:'), cyan(customConfig));
+      }
+    }
+  }
   if (argv.exclude) {
     const excludes = argv.exclude.split(',');
     for (const exclude of excludes) {
@@ -73,5 +86,7 @@ clean.description = 'Clean up various cache and output directories';
 clean.flags = {
   'dry_run': 'Do a dry run without actually deleting anything',
   'include_subpackages': 'Also clean up inner node_modules package directories',
+  'include_custom_configs':
+    'Also clean up custom config files from build-system/global-configs',
   'exclude': 'Comma-separated list of directories to exclude from deletion',
 };
diff --git a/build-system/tasks/prepend-global/index.js b/build-system/tasks/prepend-global/index.js
@@ -10,7 +10,7 @@ const {log} = require('../../common/logging');
 
 const exec = util.promisify(childProcess.exec);
 
-const {cyan, red} = colors;
+const {cyan, red, yellow} = colors;
 
 /**
  * List of unminified targets to which AMP_CONFIG should be written
@@ -22,10 +22,13 @@ const UNMINIFIED_TARGETS = ['alp.max', 'amp-inabox', 'amp-shadow', 'amp'];
  */
 const MINIFIED_TARGETS = ['alp', 'amp4ads-v0', 'shadow-v0', 'v0'];
 
-// custom-config.json overlays the active config. It is not part of checked-in
-// source (.gitignore'd). See:
-// https://github.com/ampproject/amphtml/blob/main/build-system/global-configs/README.md#custom-configjson
-const customConfigFile = 'build-system/global-configs/custom-config.json';
+/**
+ * Path to custom overlay config, see: build-system/global-configs/README.md
+ */
+const CUSTOM_OVERLAY_CONFIG_PATH = path.resolve(
+  __dirname,
+  '../../global-configs/custom-config.json'
+);
 
 /**
  * Returns the number of AMP_CONFIG matches in the given config string.
@@ -143,9 +146,6 @@ async function getConfig(
     opt_localBranch,
     opt_branch
   );
-  const overlayString = await fs.promises
-    .readFile(customConfigFile, 'utf8')
-    .catch(() => {});
 
   let configJson;
   try {
@@ -154,16 +154,20 @@ async function getConfig(
     log(red(`Error parsing config file: ${filename}`));
     throw e;
   }
-  if (overlayString) {
+
+  if (fs.existsSync(CUSTOM_OVERLAY_CONFIG_PATH)) {
+    const overlayFilename = path.basename(CUSTOM_OVERLAY_CONFIG_PATH);
     try {
-      const overlayJson = JSON.parse(overlayString);
+      const overlayJson = require(CUSTOM_OVERLAY_CONFIG_PATH);
       Object.assign(configJson, overlayJson);
-      log('Overlaid config with', cyan(path.basename(customConfigFile)));
-    } catch (e) {
       log(
-        red('Could not apply overlay from'),
-        cyan(path.basename(customConfigFile))
+        yellow('Notice:'),
+        cyan(type),
+        'config overlaid with',
+        cyan(overlayFilename)
       );
+    } catch (e) {
+      log(red('Could not apply overlay from'), cyan(overlayFilename));
     }
   }
   if (opt_localDev) {

diff --git a/build-system/tasks/release/index.js b/build-system/tasks/release/index.js
@@ -37,7 +37,7 @@ const fs = require('fs-extra');
 const klaw = require('klaw');
 const path = require('path');
 const tar = require('tar');
-const {cyan, green} = require('kleur/colors');
+const {cyan, green, red, yellow} = require('kleur/colors');
 const {execOrDie} = require('../../common/exec');
 const {log} = require('../../common/logging');
 const {MINIFIED_TARGETS} = require('../prepend-global');
@@ -116,6 +116,22 @@ const CHANNEL_CONFIGS = {
   '25': {type: 'experimentC', configBase: 'prod'}, // Spec name: 'inabox-experimentC'
 };
 
+/**
+ * Path to custom flavors config, see: build-system/global-configs/README.md
+ */
+const CUSTOM_FLAVORS_CONFIG_PATH = path.resolve(
+  __dirname,
+  '../../global-configs/custom-flavors-config.json'
+);
+
+/**
+ * Path to custom overlay config, see: build-system/global-configs/README.md
+ */
+const CUSTOM_OVERLAY_CONFIG_PATH = path.resolve(
+  __dirname,
+  '../../global-configs/custom-config.json'
+);
+
 /**
  * Prints a separator line so logs are easy to read.
  */
@@ -141,11 +157,27 @@ async function prepareEnvironment_(outputDir, tempDir) {
  * Discovers which AMP flavors are defined in the current working directory.
  *
  * The returned list of flavors will always contain the base flavor, and any
- * defined experiments in ../../global-configs/experiments-config.json.
+ * defined experiments in ../../global-configs/experiments-config.json, as well
+ * as custom flavors in ../../global-configs/custom-flavors-config.json.
  *
  * @return {!Array<!DistFlavorDef>} list of AMP flavors to build.
  */
 function discoverDistFlavors_() {
+  let customFlavorsConfig = [];
+  if (fs.existsSync(CUSTOM_FLAVORS_CONFIG_PATH)) {
+    const flavorsFilename = path.basename(CUSTOM_FLAVORS_CONFIG_PATH);
+    try {
+      customFlavorsConfig = require(CUSTOM_FLAVORS_CONFIG_PATH);
+      log(
+        yellow('Notice:'),
+        'release flavors supplemented by',
+        cyan(flavorsFilename)
+      );
+    } catch (ex) {
+      log(red('Could not load custom flavors from:'), cyan(flavorsFilename));
+    }
+  }
+
   const experimentConfigDefs = Object.entries(experimentsConfig);
   const distFlavors = [
     BASE_FLAVOR_CONFIG,
@@ -171,6 +203,7 @@ function discoverDistFlavors_() {
           ...experimentConfig,
         })
       ),
+    ...customFlavorsConfig,
   ].filter(
     // If --flavor is defined, filter out the rest.
     ({flavorType}) => !argv.flavor || flavorType == argv.flavor
@@ -382,11 +415,29 @@ async function prependConfig_(outputDir) {
   for (const [rtvPrefix, channelConfig] of activeChannels) {
     const rtvNumber = `${rtvPrefix}${VERSION}`;
     const rtvPath = path.join(outputDir, 'org-cdn/rtv', rtvNumber);
+    let overlayConfig = {};
+    if (fs.existsSync(CUSTOM_OVERLAY_CONFIG_PATH)) {
+      const overlayFilename = path.basename(CUSTOM_OVERLAY_CONFIG_PATH);
+      try {
+        overlayConfig = require(CUSTOM_OVERLAY_CONFIG_PATH);
+        log(
+          yellow('Notice:'),
+          cyan(channelConfig.configBase),
+          'config overlaid with',
+          cyan(overlayFilename)
+        );
+      } catch (ex) {
+        log(red('Could not apply overlay from'), cyan(overlayFilename));
+      }
+    }
+
     const channelPartialConfig = {
       v: rtvNumber,
       type: channelConfig.type,
       ...require(`../../global-configs/${channelConfig.configBase}-config.json`),
+      ...overlayConfig,
     };
+
     // Mapping of entry file names to a dictionary of AMP_CONFIG additions.
     const targetsToConfig = MINIFIED_TARGETS.flatMap((minifiedTarget) => {
       const targets = [];
@@ -439,7 +490,7 @@ async function populateNetWildcard_(tempDir, outputDir) {
 }
 
 /**
- * Cleans are deletes the temp directory.
+ * Cleans and deletes the temp directory.
  *
  * @param {string} tempDir full directory path to temporary working directory.
  * @return {Promise<void>}

diff --git a/docs/spec/amp-framework-hosting.md b/docs/spec/amp-framework-hosting.md
@@ -48,15 +48,15 @@ The AMP framework can be built from source or downloaded pre-built. Building the
 
 ### Option 1: Build the framework yourself
 
-Refer to the [Developing in AMP](https://github.com/ampproject/amphtml/blob/main/docs/developing.md) guide to familiarize yourself with building and testing the AMP framework. Once you are comfortable with the build system, a few small changes will customize the framework to run from your host.
+Refer to the [Developing in AMP](../developing.md) guide to familiarize yourself with building and testing the AMP framework. Once you are comfortable with the build system, a few small changes will customize the framework to run from your host.
 
 #### Update URLs config
 
-When AMP is built, several scripts are prepended with an `AMP_CONFIG` environment variable (object) containing basic information like: runtime version, config type, experiment enable/disable status, etc. This object can be customized at build time to inform the runtime where the framework is hosted. See [build-system/global-configs/README.md](https://github.com/ampproject/amphtml/tree/main/build-system/global-configs#custom-configjson) for information about the `custom-config.json` overlay.
+When AMP is built, several scripts are prepended with an `AMP_CONFIG` environment variable (object) containing basic information like: runtime version, config type, experiment enable/disable status, etc. This object can be customized at build time to inform the runtime where the framework is hosted. See [build-system/global-configs/README.md](../../build-system/global-configs/README.md#custom-configjson) for information about the `custom-config.json` overlay.
 
 Create JSON file `build-system/global-configs/custom-config.json` with the following contents:
 
-```
+```json
 {
   "cdnUrl": "https://example.com/amp-framework",
   "geoApiUrl": "https://example.com/geo-api"
@@ -68,24 +68,47 @@ where
 -   `cdnUrl` is the base URL to your AMP framework. Defaults to `https://cdn.ampproject.org`.
 -   `geoApiUrl` (optional) is your amp-geo fallback API URL. This API is described in section [amp-geo hotpatching](#amp-geo-hotpatching). Defaults to `null`.
 
-Important: `build-system/global-configs/custom-config.json` is not part of checked-in source. If it exists, it _always_ applies at build time, overlaying the active config. Don't forget about it! You can verify the overlay applies by looking for log line `Overlaid config with custom-config.json` during the build process.
+Important: `build-system/global-configs/custom-config.json` is not part of checked-in source. If it exists, it _always_ applies at build time, overlaying the active config. Don't forget about it! The build system emits warnings like `Notice: prod config overlaid with custom-config.json` to remind you that your build will differ from the default.
 
-#### Build the framework
+#### Define a custom release flavor
 
-Build an AMP release with
+Release flavors define the runtime version prefix(es) that should be built and the command line that should be used to build each runtime version. When the build system prepares a release, it supplements the built runtime with static files and pre-compiled resources. See [build-system/global-configs/README.md](../../build-system/global-configs/README.md#custom-flavors-configjson) for information about adding a custom release flavor.
 
-```
-amp dist
-```
+Create JSON file `build-system/global-configs/custom-flavors-config.json` with the following contents:
+
+```json
+[
+  {
+    "flavorType": "self-host-prod",
+    "name": "Self-hosted production release",
+    "environment": "AMP",
+    "rtvPrefixes": [ "01" ],
+    "command": "amp dist --noconfig"
+  }
+]
 
-The built framework can be found in directory `dist`. The version assigned to the build is in `dist/version.txt` and a listing of all files included in build is in `dist/files.txt`. The framework is ready to be moved to and served from your host.
+```
 
-If you have advanced hosting capabilities or would like to manually assign a version, `amp dist` accepts these flags (among others):
+If your AMP runtime will be built with code customizations, consider using flag `--sourcemap_url` with `amp dist`:
 
--   `--config`: Indicate the release type, production (`prod`) or canary (`canary`). Defaults to `prod`.
--   `--version_override`: Assign a version to the distribution. The version must consist of 13-digits. Defaults to the latest git commit time of the active branch.
 -   `--sourcemap_url`: Provide the base URL for JavaScript source map links. This URL should contain placeholder `{version}` that will be replaced with the actual version when the AMP framework is built, for example `https://raw.githubusercontent.com/<github-username>/amphtml/{version}/`. Defaults to `https://raw.githubusercontent.com/ampproject/amphtml/{version}/`.
 
+**Tips:**
+
+-   Be sure to pass flag `--noconfig` to `amp dist` in the flavor command, otherwise you will end up with multiple `AMP_CONFIG` definitions in entrypoint files (`v0.js`, `shadow-v0.js`, etc.).
+-   Flag `--version_override` is not supported.
+-   `build-system/global-configs/custom-flavors-config.json` is not part of checked-in source. If it exists, the custom flavors are automatically made available to `amp release`.
+
+#### Build the framework
+
+Build an AMP release with
+
+```sh
+amp release --flavor="self-host-prod"
+```
+
+The built framework can be found in directory `release/org-cdn/rtv/<rtv>/`. The version assigned to the build is in `version.txt` and a listing of all files included in build is in `files.txt`. The framework is ready to be moved to and served from your host.
+
 ### Option 2: Download the framework with an AMP Toolbox tool
 
 [AMP Toolbox](https://github.com/ampproject/amp-toolbox) has both a Node.js module and a command line tool that will fetch a complete AMP framework from `cdn.ampproject.org`. Pick the tool best suited to your release workflow.
@@ -95,7 +118,7 @@ If you have advanced hosting capabilities or would like to manually assign a ver
 
 ### Option 3: Manually copy the framework from cdn.ampproject.org
 
-The AMP framework can be copied from `cdn.ampproject.org`. The latest weekly release is always served from the root of `cdn.ampproject.org`. All [non-deprecated releases](https://github.com/ampproject/amphtml/blob/main/docs/spec/amp-versioning-policy.md#version-deprecations) can be found in versioned URLs: `cdn.ampproject.org/rtv/<rtv>`, where `<rtv>` is the runtime version.
+The AMP framework can be copied from `cdn.ampproject.org`. The latest weekly release is always served from the root of `cdn.ampproject.org`. All [non-deprecated releases](./amp-versioning-policy.md#version-deprecations) can be found in versioned URLs: `cdn.ampproject.org/rtv/<rtv>`, where `<rtv>` is the runtime version.
 
 Note: The AMP Project is looking into options for packaging releases ([#27726](https://github.com/ampproject/amphtml/issues/27726)).
 
@@ -227,7 +250,7 @@ The properties are defined as follows:
 -   `ampCssUrl` (optional) is a URL to the boilerplate CSS for the current stable runtime version.
 -   `canaryPercentage` (optional) indicates the fraction of users who receive the experimental runtime version of the AMP framework instead of the current stable runtime version.
 -   `diversions` (optional) lists active non-stable runtime versions.
--   `ltsRuntimeVersion` (optional) is the current [long-term stable](https://github.com/ampproject/amphtml/blob/main/docs/lts-release.md) runtime version.
+-   `ltsRuntimeVersion` (optional) is the current [long-term stable](../lts-release.md) runtime version.
 -   `ltsCssUrl` (optional) is a URL to the boilerplate CSS for the current long-term stable runtime version.
 
 ### amp-geo hotpatching
@@ -257,7 +280,7 @@ If location detection and file modification at time of delivery are not possible
 
 The API must meet the following requirements:
 
--   Satisfy [CORS security in AMP](https://github.com/ampproject/amphtml/blob/main/docs/spec/amp-cors-requests.md)
+-   Satisfy [CORS security in AMP](./amp-cors-requests.md)
 -   Be secure (HTTPS)
 -   Return `application/json` content conforming to the following schema:
     ```
@@ -310,7 +333,7 @@ There are trade-offs in accuracy and performance when you set the client cache t
 In addition to following [TLS best practices](https://infosec.mozilla.org/guidelines/web_security), consider the following headers when hosting the AMP framework:
 
 -   `content-security-policy`: If your pages implement [AMP's CSP](https://amp.dev/documentation/guides-and-tutorials/optimize-and-measure/secure-pages/), apply a matching content security policy to your hosted framework responses. Inspect the headers on `https://cdn.ampproject.org/v0.js` for a base policy that should be expanded to include resources served from your host.
--   `access-control-allow-origin`: Some runtime components are fetched via XHR. If your AMP pages will be served from a different host than your framework, be sure to include CORS headers (see also [CORS Requests in AMP](https://github.com/ampproject/amphtml/blob/main/docs/spec/amp-cors-requests.md)).
+-   `access-control-allow-origin`: Some runtime components are fetched via XHR. If your AMP pages will be served from a different host than your framework, be sure to include CORS headers (see also [CORS Requests in AMP](./amp-cors-requests.md)).
 -   `content-type`: There are a few resources served without file extensions, or with extensions that may not be recognized by all web servers. In addition to [common types](https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Common_types), you may want to include special handling for the following:
 
     -   `/rtv/metadata` - `application/json`