🏗Allow gulp tasks to programmatically build the runtime (#28326)

build-system/babel-config/pre-closure-config.js

@@ -25,6 +25,10 @@ const {getExperimentConstant, getReplacePlugin} = require('./helpers');
  */
 function getPreClosureConfig() {
   const isCheckTypes = argv._.includes('check-types');
+  const testTasks = ['e2e', 'integration', 'visual-diff'];
+  const isTestTask = testTasks.some((task) => argv._.includes(task));
+  const isFortesting = argv.fortesting || isTestTask;
+
   // For experiment, remove FixedLayer import from v0.js, otherwise remove
   // from amp-viewer-integration
   const fixedLayerImport =
@@ -89,13 +93,13 @@ function getPreClosureConfig() {
     !isCheckTypes
       ? './build-system/babel-plugins/babel-plugin-transform-json-configuration'
       : null,
-    !(argv.fortesting || isCheckTypes)
+    !(isFortesting || isCheckTypes)
       ? [
           './build-system/babel-plugins/babel-plugin-amp-mode-transformer',
           {isEsmBuild: !!argv.esm},
         ]
       : null,
-    !(argv.fortesting || isCheckTypes)
+    !(isFortesting || isCheckTypes)
       ? './build-system/babel-plugins/babel-plugin-is_dev-constant-transformer'
       : null,
   ].filter(Boolean);

build-system/common/utils.js

@@ -19,6 +19,9 @@ const fs = require('fs-extra');
 const globby = require('globby');
 const log = require('fancy-log');
 const path = require('path');
+const {clean} = require('../tasks/clean');
+const {doBuild} = require('../tasks/build');
+const {doDist} = require('../tasks/dist');
 const {execOrDie} = require('./exec');
 const {gitDiffNameOnlyMaster} = require('./git');
 const {green, cyan, yellow} = require('ansi-colors');
@@ -27,24 +30,16 @@ const {isTravisBuild} = require('./travis');
 const ROOT_DIR = path.resolve(__dirname, '../../');
 
 /**
- * Cleans and builds binaries with --fortesting flag and
- * overriden config.
- *
- * @param {boolean} minified
+ * Performs a clean build of the AMP runtime in testing mode.
+ * Used by `gulp e2e|integration|visual_diff`.
  */
-// TODO(gh/amphtml/28312): Directly call dist() or build()
-// instead of spwaning a new process.
-function buildRuntime(minified = true) {
-  execOrDie('gulp clean');
-
-  let command = minified ? `gulp dist --fortesting` : `gulp build --fortesting`;
-  if (argv.core_runtime_only) {
-    command += ` --core_runtime_only`;
-  } else if (argv.extensions) {
-    command += ` --extensions=${argv.extensions}`;
+async function buildRuntime() {
+  await clean();
+  if (argv.compiled) {
+    await doDist({fortesting: true});
+  } else {
+    await doBuild({fortesting: true});
   }
-
-  execOrDie(command);
 }
 
 /**

build-system/compile/compile.js

@@ -396,7 +396,7 @@ function compile(
         )
         .pipe(postClosureBabel())
         .pipe(sanitize())
-        .pipe(writeSourcemaps())
+        .pipe(writeSourcemaps(options))
         .pipe(gulp.dest('.'))
         .on('end', resolve);
     }

build-system/compile/helpers.js

@@ -21,23 +21,23 @@ const {VERSION: internalRuntimeVersion} = require('./internal-version');
 
 const argv = minimist(process.argv.slice(2));
 
-function getSourceMapBase() {
+function getSourceMapBase(options) {
   if (argv.sourcemap_url) {
     // Custom sourcemap URLs have placeholder {version} that should be
     // replaced with the actual version. Also, ensure trailing slash exists.
     return String(argv.sourcemap_url)
       .replace(/\{version\}/g, internalRuntimeVersion)
       .replace(/([^/])$/, '$1/');
   }
-  if (argv.fortesting) {
+  if (options.fortesting) {
     return 'http://localhost:8000/';
   }
   return `https://raw.githubusercontent.com/ampproject/amphtml/${internalRuntimeVersion}/`;
 }
 
-function writeSourcemaps() {
+function writeSourcemaps(options) {
   return sourcemaps.write('.', {
-    sourceRoot: getSourceMapBase(),
+    sourceRoot: getSourceMapBase(options),
     includeContent: !!argv.full_sourcemaps,
   });
 }

build-system/compile/single-pass.js

@@ -527,7 +527,7 @@ exports.singlePassCompile = async function (entryModule, options, timeInfo) {
       externs: options.externs,
       hideWarningsFor: options.hideWarningsFor,
     })
-    .then(compile)
+    .then((flagsArray) => compile(flagsArray, options))
     .then(wrapMainBinaries)
     .then(intermediateBundleConcat)
     .then(eliminateIntermediateBundles)
@@ -690,7 +690,7 @@ function cleanupWeakModuleFiles() {
   return del([weakModuleJsFile, weakModuleMapFile]);
 }
 
-function compile(flagsArray) {
+function compile(flagsArray, options) {
   log('Minifying single-pass JS with', colors.cyan('closure-compiler') + '...');
   // TODO(@cramforce): Run the post processing step
   return new Promise(function (resolve, reject) {
@@ -703,7 +703,7 @@ function compile(flagsArray) {
       .on('error', (err) => handleSinglePassCompilerError(err))
       .pipe(gulpIf(!argv.pseudo_names, checkForUnknownDeps()))
       .on('error', reject)
-      .pipe(writeSourcemaps())
+      .pipe(writeSourcemaps(options))
       .pipe(
         gulpIf(
           /(\/amp-|\/_base)/,

build-system/pr-check/visual-diff-tests.js

@@ -44,7 +44,7 @@ function main() {
     downloadDistOutput(FILENAME);
     timedExecOrDie('gulp update-packages');
     process.env['PERCY_TOKEN'] = atob(process.env.PERCY_TOKEN_ENCODED);
-    timedExecOrDie('gulp visual-diff --nobuild --master');
+    timedExecOrDie('gulp visual-diff --compiled --nobuild --master');
   } else {
     printChangeSummary(FILENAME);
     const buildTargets = determineBuildTargets(FILENAME);
@@ -56,9 +56,9 @@ function main() {
     ) {
       downloadDistOutput(FILENAME);
       timedExecOrDie('gulp update-packages');
-      timedExecOrDie('gulp visual-diff --nobuild');
+      timedExecOrDie('gulp visual-diff --compiled --nobuild');
     } else {
-      timedExecOrDie('gulp visual-diff --nobuild --empty');
+      timedExecOrDie('gulp visual-diff --empty');
       console.log(
         `${FILELOGPREFIX} Skipping`,
         colors.cyan('Visual Diff Tests'),

build-system/tasks/build.js

@@ -62,30 +62,44 @@ async function watch() {
  * Perform the prerequisite steps before starting the unminified build.
  * Used by `gulp` and `gulp build`.
  *
- * @param {boolean} watch
+ * @param {!Object} options
  */
-async function runPreBuildSteps(watch) {
-  await compileCss(watch);
+async function runPreBuildSteps(options) {
+  await compileCss(options);
   await compileJison();
-  await bootstrapThirdPartyFrames(watch);
+  await bootstrapThirdPartyFrames(options);
 }
 
 /**
  * Unminified build. Entry point for `gulp build`.
  */
 async function build() {
+  await doBuild();
+}
+
+/**
+ * Performs an unminified build with the given extra args.
+ *
+ * @param {Object=} extraArgs
+ */
+async function doBuild(extraArgs = {}) {
   maybeUpdatePackages();
   const handlerProcess = createCtrlcHandler('build');
   process.env.NODE_ENV = 'development';
+  const options = {
+    fortesting: extraArgs.fortesting || argv.fortesting,
+    minify: false,
+    watch: argv.watch,
+  };
   printNobuildHelp();
   printConfigHelp('gulp build');
   parseExtensionFlags();
-  await runPreBuildSteps(argv.watch);
+  await runPreBuildSteps(options);
   if (argv.core_runtime_only) {
-    await compileCoreRuntime(argv.watch, /* minify */ false);
+    await compileCoreRuntime(options);
   } else {
-    await compileAllJs(/* minify */ false);
-    await buildExtensions({watch: argv.watch});
+    await compileAllJs(options);
+    await buildExtensions(options);
   }
   if (!argv.watch) {
     exitCtrlcHandler(handlerProcess);
@@ -94,6 +108,7 @@ async function build() {
 
 module.exports = {
   build,
+  doBuild,
   runPreBuildSteps,
   watch,
 };

build-system/tasks/css.js

@@ -76,11 +76,12 @@ const cssEntryPoints = [
 
 /**
  * Compile all the css and drop in the build folder
- * @param {boolean} watch
+ *
+ * @param {Object=} options
  * @return {!Promise}
  */
-function compileCss(watch) {
-  if (watch) {
+function compileCss(options = {}) {
+  if (options.watch) {
     const watchFunc = () => {
       compileCss();
     };

build-system/tasks/dist.js

@@ -74,14 +74,16 @@ const hostname = argv.hostname || 'cdn.ampproject.org';
 
 /**
  * Prints a useful help message prior to the gulp dist task
+ *
+ * @param {!Object} options
  */
-function printDistHelp() {
+function printDistHelp(options) {
   if (argv.sanitize_vars_for_diff && !argv.pseudo_names) {
     throw new Error('--sanitize_vars_for_diff requires --pseudo_names');
   }
 
   let cmd = 'gulp dist';
-  if (argv.fortesting) {
+  if (options.fortesting) {
     cmd = cmd + ' --fortesting';
   }
   if (argv.single_pass) {
@@ -103,42 +105,54 @@ function printDistHelp() {
  * Perform the prerequisite steps before starting the minified build.
  * Used by `gulp` and `gulp dist`.
  *
- * @param {boolean} watch
+ * @param {!Object} options
  */
-async function runPreDistSteps(watch) {
+async function runPreDistSteps(options) {
   cleanupBuildDir();
   await prebuild();
-  await compileCss(watch);
+  await compileCss(options);
   await compileJison();
   await copyCss();
   await copyParsers();
-  await bootstrapThirdPartyFrames(watch, /* minify */ true);
+  await bootstrapThirdPartyFrames(options);
   await startNailgunServer(distNailgunPort, /* detached */ false);
   displayLifecycleDebugging();
 }
 
 /**
- * Dist Build
- * @return {!Promise}
+ * Minified build. Entry point for `gulp dist`.
  */
 async function dist() {
+  await doDist();
+}
+
+/**
+ * Performs a minified build with the given extra args.
+ *
+ * @param {Object=} extraArgs
+ */
+async function doDist(extraArgs = {}) {
   maybeUpdatePackages();
   const handlerProcess = createCtrlcHandler('dist');
   process.env.NODE_ENV = 'production';
+  const options = {
+    fortesting: extraArgs.fortesting || argv.fortesting,
+    minify: true,
+    watch: argv.watch,
+  };
   printNobuildHelp();
-  printDistHelp();
-
-  await runPreDistSteps(argv.watch);
+  printDistHelp(options);
+  await runPreDistSteps(options);
 
   // Steps that use closure compiler. Small ones before large (parallel) ones.
   if (argv.core_runtime_only) {
-    await compileCoreRuntime(argv.watch, /* minify */ true);
+    await compileCoreRuntime(options);
   } else {
     await buildExperiments();
     await buildLoginDone('0.1');
     await buildWebPushPublisherFiles();
-    await compileAllJs(/* minify */ true);
-    await buildExtensions({minify: true, watch: argv.watch});
+    await compileAllJs(options);
+    await buildExtensions(options);
   }
   if (!argv.watch) {
     await stopNailgunServer(distNailgunPort);
@@ -435,6 +449,7 @@ function preBuildLoginDoneVersion(version) {
 
 module.exports = {
   dist,
+  doDist,
   runPreDistSteps,
 };
 

build-system/tasks/e2e/index.js

@@ -39,11 +39,11 @@ const PORT = 8000;
 const SLOW_TEST_THRESHOLD_MS = 2500;
 const TEST_RETRIES = isTravisBuild() ? 2 : 0;
 
-async function launchWebServer_(minified) {
+async function launchWebServer_() {
   await startServer(
     {host: HOST, port: PORT},
     {quiet: !argv.debug},
-    {compiled: minified}
+    {compiled: argv.compiled}
   );
 }
 
@@ -84,11 +84,11 @@ async function e2e() {
 
   // build runtime
   if (!argv.nobuild) {
-    buildRuntime(/* minified */ !!argv.compiled);
+    await buildRuntime();
   }
 
   // start up web server
-  await launchWebServer_(/* minified */ argv.compiled);
+  await launchWebServer_();
 
   // run tests
   if (!argv.watch) {
@@ -154,7 +154,7 @@ e2e.flags = {
   'nobuild':
     '  Skips building the runtime via `gulp (build|dist) --fortesting`',
   'extensions': '  Builds only the listed extensions.',
-  'compiled': '  Runs the tests using minified js',
+  'compiled': '  Runs tests against minified JS',
   'files': '  Run tests found in a specific path (ex: **/test-e2e/*.js)',
   'testnames': '  Lists the name of each test being run',
   'watch': '  Watches for changes in files, runs corresponding test(s)',