🏗 Split experiment builds and tests into separate jobs (#32614)

.circleci/config.yml

@@ -225,9 +225,36 @@ jobs:
           name: 'Performance Tests'
           command: node build-system/pr-check/performance-tests.js
       - fail_fast
-  'Experiment A Tests':
+  'Experiment A Build':
+    executor:
+      name: amphtml-xlarge-executor
+    steps:
+      - setup_vm
+      - run:
+          name: 'Experiment A Build'
+          command: node build-system/pr-check/experiment-build.js --experiment=experimentA
+      - fail_fast
+  'Experiment B Build':
     executor:
       name: amphtml-xlarge-executor
+    steps:
+      - setup_vm
+      - run:
+          name: 'Experiment B Build'
+          command: node build-system/pr-check/experiment-build.js --experiment=experimentB
+      - fail_fast
+  'Experiment C Build':
+    executor:
+      name: amphtml-xlarge-executor
+    steps:
+      - setup_vm
+      - run:
+          name: 'Experiment C Build'
+          command: node build-system/pr-check/experiment-build.js --experiment=experimentC
+      - fail_fast
+  'Experiment A Tests':
+    executor:
+      name: amphtml-large-executor
     steps:
       - setup_vm
       - install_chrome
@@ -241,7 +268,7 @@ jobs:
       - fail_fast
   'Experiment B Tests':
     executor:
-      name: amphtml-xlarge-executor
+      name: amphtml-large-executor
     steps:
       - setup_vm
       - install_chrome
@@ -255,7 +282,7 @@ jobs:
       - fail_fast
   'Experiment C Tests':
     executor:
-      name: amphtml-xlarge-executor
+      name: amphtml-large-executor
     steps:
       - setup_vm
       - install_chrome
@@ -309,12 +336,24 @@ workflows:
           <<: *push_and_pr_builds
           requires:
             - 'Nomodule Build'
+      - 'Experiment A Build':
+          <<: *push_and_pr_builds
+      - 'Experiment B Build':
+          <<: *push_and_pr_builds
+      - 'Experiment C Build':
+          <<: *push_and_pr_builds
       - 'Experiment A Tests':
           <<: *push_and_pr_builds
+          requires:
+            - 'Experiment A Build'
       - 'Experiment B Tests':
           <<: *push_and_pr_builds
+          requires:
+            - 'Experiment B Build'
       - 'Experiment C Tests':
           <<: *push_and_pr_builds
+          requires:
+            - 'Experiment C Build'
       # TODO(wg-performance, #12128): This takes 30 mins and fails regularly.
       # - 'Performance Tests':
       #     <<: *push_builds_only

.travis.yml

@@ -47,8 +47,6 @@ addons:
 stages:
   - name: build
   - name: test
-  - name: experiment
-    if: type = push
 jobs:
   include:
     - stage: build
@@ -99,22 +97,4 @@ jobs:
       name: 'End to End Tests'
       script:
         - unbuffer node build-system/pr-check/e2e-tests.js
-    - stage: experiment
-      name: 'Experiment A Tests'
-      script:
-        - unbuffer node build-system/pr-check/experiment-tests.js --experiment=experimentA
-    - stage: experiment
-      name: 'Experiment B Tests'
-      script:
-        - unbuffer node build-system/pr-check/experiment-tests.js --experiment=experimentB
-    - stage: experiment
-      name: 'Experiment C Tests'
-      script:
-        - unbuffer node build-system/pr-check/experiment-tests.js --experiment=experimentC
-    - stage: experiment
-      name: 'Performance Tests'
-      script:
-        - unbuffer node build-system/pr-check/performance-tests.js
-  allow_failures:
-    - script: unbuffer node build-system/pr-check/performance-tests.js # See #28148
   fast_finish: true

build-system/pr-check/experiment-build.js

@@ -0,0 +1,66 @@
+/**
+ * Copyright 2021 The AMP HTML Authors. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS-IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+'use strict';
+
+/**
+ * @fileoverview Script that builds the experiment A/B/C runtime during CI.
+ */
+
+const {
+  getExperimentConfig,
+  printSkipMessage,
+  timedExecOrDie,
+  uploadExperimentOutput,
+} = require('./utils');
+const {buildTargetsInclude, Targets} = require('./build-targets');
+const {experiment} = require('minimist')(process.argv.slice(2));
+const {runCiJob} = require('./ci-job');
+
+const jobName = `${experiment}-build.js`;
+
+function pushBuildWorkflow() {
+  const config = getExperimentConfig(experiment);
+  if (config) {
+    const defineFlag = `--define_experiment_constant ${config.define_experiment_constant}`;
+    timedExecOrDie('gulp update-packages');
+    timedExecOrDie(`gulp dist --fortesting ${defineFlag}`);
+    uploadExperimentOutput(experiment);
+  } else {
+    printSkipMessage(
+      jobName,
+      `${experiment} is expired, misconfigured, or does not exist`
+    );
+  }
+}
+
+function prBuildWorkflow() {
+  if (
+    buildTargetsInclude(
+      Targets.RUNTIME,
+      Targets.INTEGRATION_TEST,
+      Targets.E2E_TEST
+    )
+  ) {
+    pushBuildWorkflow();
+  } else {
+    printSkipMessage(
+      jobName,
+      'this PR does not affect the runtime, integration tests, or end-to-end tests'
+    );
+  }
+}
+
+runCiJob(jobName, pushBuildWorkflow, prBuildWorkflow);

build-system/pr-check/experiment-tests.js

@@ -19,35 +19,24 @@
  * @fileoverview Script that runs the experiment A/B/C tests during CI.
  */
 
-const experimentsConfig = require('../global-configs/experiments-config.json');
+const {
+  getExperimentConfig,
+  downloadExperimentOutput,
+  printSkipMessage,
+  timedExecOrDie,
+} = require('./utils');
 const {buildTargetsInclude, Targets} = require('./build-targets');
 const {experiment} = require('minimist')(process.argv.slice(2));
-const {printSkipMessage, timedExecOrDie} = require('./utils');
 const {runCiJob} = require('./ci-job');
 
 const jobName = `${experiment}-tests.js`;
 
-/**
- * Extracts and validates the experiment config.
- * @return {Object|null}
- */
-function getConfig_() {
-  const config = experimentsConfig[experiment];
-  const valid =
-    config?.name &&
-    config?.define_experiment_constant &&
-    config?.expiration_date_utc &&
-    new Date(config.expiration_date_utc) >= Date.now();
-  return valid ? config : null;
-}
-
 function pushBuildWorkflow() {
-  const config = getConfig_();
+  const config = getExperimentConfig(experiment);
   if (config) {
     const defineFlag = `--define_experiment_constant ${config.define_experiment_constant}`;
     const experimentFlag = `--experiment ${experiment}`;
-    timedExecOrDie('gulp update-packages');
-    timedExecOrDie(`gulp dist --fortesting ${defineFlag}`);
+    downloadExperimentOutput(experiment);
     timedExecOrDie(
       `gulp integration --nobuild --compiled --headless ${experimentFlag} ${defineFlag}`
     );

build-system/pr-check/utils.js

@@ -15,6 +15,7 @@
  */
 'use strict';
 
+const experimentsConfig = require('../global-configs/experiments-config.json');
 const {
   gitBranchCreationPoint,
   gitBranchName,
@@ -33,6 +34,7 @@ const {replaceUrls} = require('../tasks/pr-deploy-bot-utils');
 const UNMINIFIED_OUTPUT_FILE = `amp_unminified_${ciBuildSha()}.zip`;
 const NOMODULE_OUTPUT_FILE = `amp_nomodule_${ciBuildSha()}.zip`;
 const MODULE_OUTPUT_FILE = `amp_module_${ciBuildSha()}.zip`;
+const EXPERIMENT_OUTPUT_FILE = (exp) => `amp_${exp}_${ciBuildSha()}.zip`;
 
 const BUILD_OUTPUT_DIRS = 'build/ dist/ dist.3p/';
 const APP_SERVING_DIRS =
@@ -285,6 +287,14 @@ function downloadModuleOutput() {
   downloadOutput_(MODULE_OUTPUT_FILE, BUILD_OUTPUT_DIRS);
 }
 
+/**
+ * Downloads and unzips output for the given experiment from storage
+ * @param {string} exp
+ */
+function downloadExperimentOutput(exp) {
+  downloadOutput_(EXPERIMENT_OUTPUT_FILE(exp), BUILD_OUTPUT_DIRS);
+}
+
 /**
  * Zips and uploads the build output to a remote storage location
  */
@@ -308,6 +318,16 @@ function uploadModuleOutput() {
   uploadOutput_(MODULE_OUTPUT_FILE, moduleOutputDirs);
 }
 
+/**
+ * Zips and uploads the output for the given experiment to a remote storage
+ * location
+ * @param {string} exp
+ */
+function uploadExperimentOutput(exp) {
+  const experimentOutputDirs = `${BUILD_OUTPUT_DIRS} ${APP_SERVING_DIRS}`;
+  uploadOutput_(EXPERIMENT_OUTPUT_FILE(exp), experimentOutputDirs);
+}
+
 /**
  * Replaces URLS in HTML files, zips and uploads nomodule output,
  * and signals to the AMP PR Deploy bot that the upload is complete.
@@ -318,11 +338,28 @@ async function processAndUploadNomoduleOutput() {
   uploadNomoduleOutput();
 }
 
+/**
+ * Extracts and validates the config for the given experiment.
+ * @param {string} experiment
+ * @return {Object|null}
+ */
+function getExperimentConfig(experiment) {
+  const config = experimentsConfig[experiment];
+  const valid =
+    config?.name &&
+    config?.define_experiment_constant &&
+    config?.expiration_date_utc &&
+    new Date(config.expiration_date_utc) >= Date.now();
+  return valid ? config : null;
+}
+
 module.exports = {
   abortTimedJob,
+  downloadExperimentOutput,
   downloadUnminifiedOutput,
   downloadNomoduleOutput,
   downloadModuleOutput,
+  getExperimentConfig,
   printChangeSummary,
   printSkipMessage,
   processAndUploadNomoduleOutput,
@@ -332,6 +369,7 @@ module.exports = {
   timedExecOrDie,
   timedExecWithError,
   timedExecOrThrow,
+  uploadExperimentOutput,
   uploadUnminifiedOutput,
   uploadNomoduleOutput,
   uploadModuleOutput,