🏗 Lazy building 3p integrations (#32772)

* Lazy building 3p integrations

* Cache bundle

* Review comments

build-system/server/lazy-build.js

@@ -16,17 +16,24 @@
 'use strict';
 
 const argv = require('minimist')(process.argv.slice(2));
+const {
+  doBuild3pVendor,
+  generateBundles,
+} = require('../tasks/3p-vendor-helpers');
 const {
   doBuildExtension,
   maybeInitializeExtensions,
   getExtensionsToBuild,
 } = require('../tasks/extension-helpers');
 const {doBuildJs, compileCoreRuntime} = require('../tasks/helpers');
 const {jsBundles} = require('../compile/bundles.config');
+const {VERSION} = require('../compile/internal-version');
 
 const extensionBundles = {};
 maybeInitializeExtensions(extensionBundles, /* includeLatest */ true);
 
+const vendorBundles = generateBundles();
+
 /**
  * Gets the unminified name of the bundle if it can be lazily built.
  *
@@ -127,6 +134,20 @@ async function lazyBuildJs(req, _res, next) {
   await lazyBuild(req.url, matcher, jsBundles, doBuildJs, next);
 }
 
+/**
+ * Lazy builds a 3p iframe vendor file when requested.
+ *
+ * @param {!Object} req
+ * @param {!Object} _res
+ * @param {function(): void} next
+ */
+async function lazyBuild3pVendor(req, _res, next) {
+  const matcher = argv.compiled
+    ? new RegExp(`\\/dist\\.3p\\/${VERSION}\\/vendor\\/([^\/]*)\\.js`) // '/dist.3p/21900000/vendor/*.js'
+    : /\/dist\.3p\/current\/vendor\/([^\/]*)\.max\.js/; // '/dist.3p/current/vendor/*.max.js'
+  await lazyBuild(req.url, matcher, vendorBundles, doBuild3pVendor, next);
+}
+
 /**
  * Pre-builds the core runtime and the JS files that it loads.
  */
@@ -152,6 +173,7 @@ async function preBuildExtensions() {
 module.exports = {
   lazyBuildExtensions,
   lazyBuildJs,
+  lazyBuild3pVendor,
   preBuildExtensions,
   preBuildRuntimeFiles,
 };

build-system/tasks/3p-vendor-helpers.js

@@ -18,10 +18,13 @@ const debounce = require('debounce');
 const globby = require('globby');
 const {compileJsWithEsbuild} = require('./helpers');
 const {endBuildStep} = require('./helpers');
+const {red, cyan} = require('kleur/colors');
 const {VERSION} = require('../compile/internal-version');
 const {watchDebounceDelay} = require('./helpers');
 const {watch} = require('chokidar');
 
+const SRCPATH = ['3p/vendors/*.js'];
+
 /**
  * Entry point for 'amp ad-vendor-configs'
  * Compile all the vendor configs and drop in the dist folder
@@ -31,41 +34,30 @@ const {watch} = require('chokidar');
 async function buildVendorConfigs(options) {
   options = options || {};
 
-  const srcPath = ['3p/vendors/*.js'];
   const destPath = 'dist.3p/';
 
-  // ignore test JS if not fortesting or build.
-  if (!options.fortesting) {
-    srcPath.push('!3p/vendors/_ping_.js');
-  }
-
   if (options.watch) {
     // Do not set watchers again when we get called by the watcher.
     const copyOptions = {...options, watch: false, calledByWatcher: true};
     const watchFunc = () => {
       buildVendorConfigs(copyOptions);
     };
-    watch(srcPath).on('change', debounce(watchFunc, watchDebounceDelay));
+    watch(SRCPATH).on('change', debounce(watchFunc, watchDebounceDelay));
   }
 
   const startTime = Date.now();
+  const bundles = generateBundles();
 
-  const filesToBuild = globby.sync(srcPath);
-  const srcMatcher = /^3p\/vendors\/(.*)\.js/;
-  const results = [];
-
-  for (const index in filesToBuild) {
-    const src = filesToBuild[index];
-    const match = src.match(srcMatcher);
-    if (!match || match.length != 2) {
-      throw new Error(`${src} is not a valid 3p vendor path`);
-    }
-
-    // Extract vendor file name
-    const name = match[1];
-    results.push(buildVendor(name, options));
-  }
-  await Promise.all(results);
+  await Promise.all(
+    Object.values(bundles).map((bundle) =>
+      compileJsWithEsbuild(
+        bundle.srcDir,
+        bundle.srcFilename,
+        options.minify ? bundle.minifiedDestDir : bundle.destDir,
+        {...bundle.options, ...options}
+      )
+    )
+  );
 
   endBuildStep(
     (options.minify ? 'Minified' : 'Compiled') +
@@ -76,33 +68,78 @@ async function buildVendorConfigs(options) {
 }
 
 /**
- * Build the JavaScript for the vendor specified
+ * Build the JavaScript for the vendor specified for lazy building.
  *
- * @param {string} name Name of the extension. Must be the sub directory in
- *     the extensions directory and the name of the JS and optional CSS file.
- * @param {!Object} options
+ * @param {!Object} jsBundles
+ * @param {string} name
+ * @param {?Object} options
  * @return {!Promise}
  */
-async function buildVendor(name, options) {
-  await compileJsWithEsbuild(
-    './3p/vendors/',
-    name + '.js',
-    './dist.3p/',
-    Object.assign(options, {
-      include3pDirectories: true,
-      includePolyfills: true,
-      externs: ['./ads/ads.extern.js'],
-      toName: `current/vendor/${name}.max.js`,
-      minifiedName: `${VERSION}/vendor/${name}.js`,
-    })
-  );
+async function doBuild3pVendor(jsBundles, name, options) {
+  const target = jsBundles[name];
+  if (target) {
+    return compileJsWithEsbuild(
+      target.srcDir,
+      target.srcFilename,
+      options.minify ? target.minifiedDestDir : target.destDir,
+      {...target.options, ...options}
+    );
+  } else {
+    return Promise.reject(
+      [red('Error:'), 'Could not find', cyan(name)].join(' ')
+    );
+  }
+}
+
+/**
+ * Generate bundles for all 3p vendors to be built.
+ * @return {Object}
+ */
+function generateBundles() {
+  const bundles = {};
+  listVendors().forEach((vendor) => {
+    bundles[`${vendor}`] = {
+      srcDir: './3p/vendors/',
+      srcFilename: `${vendor}.js`,
+      destDir: './dist.3p/current/vendor/',
+      minifiedDestDir: `./dist.3p/${VERSION}/vendor/`,
+      options: {
+        include3pDirectories: true,
+        includePolyfills: true,
+        externs: ['./ads/ads.extern.js'],
+        toName: `${vendor}.max.js`,
+        minifiedName: `${vendor}.js`,
+      },
+    };
+  });
+  return bundles;
+}
+
+/**
+ * Return all 3p iframe vendors' names.
+ * @return {!Array<string>}
+ */
+function listVendors() {
+  const filesToBuild = globby.sync(SRCPATH);
+  const srcMatcher = /^3p\/vendors\/(.*)\.js/;
+  const results = [];
+
+  for (const index in filesToBuild) {
+    const src = filesToBuild[index];
+    const match = src.match(srcMatcher);
+    if (!match || match.length != 2) {
+      throw new Error(`${src} is not a valid 3p vendor path`);
+    }
 
-  // If an incremental watch build fails, simply return.
-  if (options.errored) {
-    return;
+    // Extract vendor file name
+    const name = match[1];
+    results.push(name);
   }
+  return results;
 }
 
 module.exports = {
   buildVendorConfigs,
+  doBuild3pVendor,
+  generateBundles,
 };

build-system/tasks/serve.js

@@ -31,6 +31,7 @@ const {
 const {
   lazyBuildExtensions,
   lazyBuildJs,
+  lazyBuild3pVendor,
   preBuildRuntimeFiles,
   preBuildExtensions,
 } = require('../server/lazy-build');
@@ -74,6 +75,7 @@ function getMiddleware() {
   if (lazyBuild) {
     middleware.push(lazyBuildExtensions);
     middleware.push(lazyBuildJs);
+    middleware.push(lazyBuild3pVendor);
   }
   return middleware;
 }