tests(devtools): remove webtests, sync e2e tests, use release mode (#…

…14163)

.github/scripts/generate-devtools-hash.sh

@@ -7,14 +7,13 @@
 ##
 
 # Prints to stdout text that, when it changes, indicates that the devtools tests
-# should update its dependencies (devtools frontend, content shell, blink tools).
+# should rebuild the devtools frontend.
 
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 LH_ROOT="$SCRIPT_DIR/../.."
 
 cd "$LH_ROOT"
 bash .github/scripts/print-devtools-relevant-commits.sh
 md5 \
-  lighthouse-core/test/chromium-web-tests/* \
-  third-party/chromium-webtests/webtests/http/tests/devtools/lighthouse/**/*.* \
+  lighthouse-core/test/devtools-tests/* \
   third-party/devtools-tests/e2e/**/*.*

.github/workflows/devtools.yml

@@ -8,7 +8,6 @@ on:
 env:
   DEPOT_TOOLS_PATH: ${{ github.workspace }}/depot-tools
   DEVTOOLS_PATH: ${{ github.workspace }}/devtools-frontend
-  BLINK_TOOLS_PATH: ${{ github.workspace }}/blink_tools
 
 jobs:
   build:
@@ -35,15 +34,13 @@ jobs:
     # Caches are invalidated at least once a week, so that's the max time between testing
     # with the latest dependencies. Any commit to the DevTools repo touching Lighthouse
     # code will invalidate the cache sooner.
-    - name: Cache depot tools, devtools, blink tools and content shell
+    - name: Cache depot tools and devtools
       id: devtools-cache
       uses: actions/cache@v3
       with:
         path: |
           ${{ env.DEPOT_TOOLS_PATH }}
           ${{ env.DEVTOOLS_PATH }}
-          ${{ env.BLINK_TOOLS_PATH }}
-          ${{ github.workspace }}/lighthouse/.tmp/chromium-web-tests/content-shells
           ${{ github.workspace }}/.gclient
           ${{ github.workspace }}/.gclient_entries
         # This hash key changes:
@@ -52,8 +49,8 @@ jobs:
         # 3) every change to file in Lighthouse repo important to running these tests.
         #
         # The number is how many times this hash key was manually updated to break the cache.
-        key: ${{ runner.os }}-7-${{ env.WEEK_OF_THE_YEAR }}-${{ hashFiles('cdt-test-hash.txt') }}
-        restore-keys: ${{ runner.os }}-7
+        key: ${{ runner.os }}-8-${{ env.WEEK_OF_THE_YEAR }}-${{ hashFiles('cdt-test-hash.txt') }}
+        restore-keys: ${{ runner.os }}-8
     - name: Set GHA_DEVTOOLS_CACHE_HIT
       if: steps.devtools-cache.outputs.cache-hit == 'true'
       run: echo "GHA_DEVTOOLS_CACHE_HIT=1" >> $GITHUB_ENV
@@ -62,12 +59,12 @@ jobs:
       working-directory: ${{ github.workspace }}/lighthouse
 
     - name: Download depot tools
-      run: bash $GITHUB_WORKSPACE/lighthouse/lighthouse-core/test/chromium-web-tests/download-depot-tools.sh
+      run: bash $GITHUB_WORKSPACE/lighthouse/lighthouse-core/test/devtools-tests/download-depot-tools.sh
     - name: Download DevTools Frontend
-      run: bash $GITHUB_WORKSPACE/lighthouse/lighthouse-core/test/chromium-web-tests/download-devtools.sh
+      run: bash $GITHUB_WORKSPACE/lighthouse/lighthouse-core/test/devtools-tests/download-devtools.sh
 
     - name: Roll Lighthouse + build DevTools
-      run: bash $GITHUB_WORKSPACE/lighthouse/lighthouse-core/test/chromium-web-tests/roll-devtools.sh
+      run: bash $GITHUB_WORKSPACE/lighthouse/lighthouse-core/test/devtools-tests/roll-devtools.sh
 
     - name: Cache build artifacts
       id: devtools-build-artifacts
@@ -101,7 +98,7 @@ jobs:
       run: echo "$DEPOT_TOOLS_PATH" >> $GITHUB_PATH
     # For vpython.
     - name: Download depot tools
-      run: bash $GITHUB_WORKSPACE/lighthouse/lighthouse-core/test/chromium-web-tests/download-depot-tools.sh
+      run: bash $GITHUB_WORKSPACE/lighthouse/lighthouse-core/test/devtools-tests/download-depot-tools.sh
 
     - name: Load build artifacts
       id: devtools-build-artifacts
@@ -116,49 +113,6 @@ jobs:
     - name: Run e2e tests
       run: bash $GITHUB_WORKSPACE/lighthouse/lighthouse-core/test/devtools-tests/run-e2e-tests.sh
 
-  web-tests:
-    needs: [build]
-    runs-on: macos-latest
-
-    steps:
-    - name: git clone
-      uses: actions/checkout@v2
-      with:
-        path: lighthouse
-
-    - name: Use Node.js 14.x
-      uses: actions/setup-node@v1
-      with:
-        node-version: 14.x
-
-    - name: Load build artifacts
-      id: devtools-build-artifacts
-      uses: actions/cache@v3
-      with:
-        path: |
-          ${{ env.DEVTOOLS_PATH }}
-          ${{ github.workspace }}/.gclient
-          ${{ github.workspace }}/.gclient_entries
-        key: devtools-build-artifacts-${{ github.run_id }}
-
-    - name: Download Blink Tools
-      run: bash $GITHUB_WORKSPACE/lighthouse/lighthouse-core/test/chromium-web-tests/download-blink-tools.sh
-    - name: Download Content Shell
-      run: bash $GITHUB_WORKSPACE/lighthouse/lighthouse-core/test/chromium-web-tests/download-content-shell.sh
-
-    - name: Install python deps
-      run: python3 -m pip install six requests
-
-    - name: Run Web Tests
-      run: bash $GITHUB_WORKSPACE/lighthouse/lighthouse-core/test/chromium-web-tests/run-web-tests.sh
-
-    - name: Upload results
-      uses: actions/upload-artifact@v2
-      if: failure()
-      with:
-        name: results
-        path: ${{ github.workspace }}/lighthouse/.tmp/layout-test-results
-
   smoke:
     needs: [build]
     strategy:
@@ -190,11 +144,6 @@ jobs:
           ${{ github.workspace }}/.gclient_entries
         key: devtools-build-artifacts-${{ github.run_id }}
 
-    - name: Download Blink Tools
-      run: bash $GITHUB_WORKSPACE/lighthouse/lighthouse-core/test/chromium-web-tests/download-blink-tools.sh
-    - name: Download Content Shell
-      run: bash $GITHUB_WORKSPACE/lighthouse/lighthouse-core/test/chromium-web-tests/download-content-shell.sh
-
     - run: yarn --frozen-lockfile --network-timeout 1000000
       working-directory: ${{ github.workspace }}/lighthouse
 

.npmignore

@@ -35,7 +35,7 @@ timings-data/
 # ignore all folders named as such
 node_modules
 latest-run
-chromium-webtests
+devtools-tests
 .tmp
 
 # generated files

lighthouse-cli/test/smokehouse/lighthouse-runners/devtools.js

@@ -21,18 +21,18 @@ const devtoolsDir =
  * @param {string[]} logs
  * @param {string} command
  * @param {string[]} args
+ * @return {Promise<boolean>} true if command exited successfully.
  */
-async function spawnAndLog(logs, command, args) {
-  /** @type {Promise<void>} */
-  const promise = new Promise((resolve) => {
+function spawnAndLog(logs, command, args) {
+  return new Promise((resolve) => {
     const spawnHandle = spawn(command, args);
     spawnHandle.on('close', code => {
       if (code) {
         logs.push(`[FAILURE] Command exited with code: ${code}\n`);
       } else {
         logs.push('[SUCCESS] Command exited with code: 0\n');
       }
-      resolve();
+      resolve(code === 0);
     });
     spawnHandle.on('error', (error) => {
       logs.push(`ERROR: ${error.toString()}`);
@@ -48,21 +48,22 @@ async function spawnAndLog(logs, command, args) {
       logs.push(`STDERR: ${data}`);
     });
   });
-  await promise;
 }
 
-/** @type {Promise<void>} */
+/** @type {Promise<boolean>} */
 let buildDevtoolsPromise;
 /**
- * @param {string[]} logs
  * Download/pull latest DevTools, build Lighthouse for DevTools, roll to DevTools, and build DevTools.
+ * @param {string[]} logs
  */
 async function buildDevtools(logs) {
-  if (process.env.CI) return;
+  if (process.env.CI) return true;
 
   process.env.DEVTOOLS_PATH = devtoolsDir;
-  await spawnAndLog(logs, 'bash', ['lighthouse-core/test/chromium-web-tests/download-devtools.sh']);
-  await spawnAndLog(logs, 'bash', ['lighthouse-core/test/chromium-web-tests/roll-devtools.sh']);
+  const success =
+    await spawnAndLog(logs, 'bash', ['lighthouse-core/test/devtools-tests/download-devtools.sh']) &&
+    await spawnAndLog(logs, 'bash', ['lighthouse-core/test/devtools-tests/roll-devtools.sh']);
+  return success;
 }
 
 /**
@@ -80,7 +81,10 @@ async function runLighthouse(url, configJson, testRunnerOptions = {}) {
   const logs = [];
 
   if (!buildDevtoolsPromise) buildDevtoolsPromise = buildDevtools(logs);
-  await buildDevtoolsPromise;
+  if (!await buildDevtoolsPromise) {
+    const log = logs.join('') + '\n';
+    throw new Error(`failed to build devtools:\n${log}`);
+  }
 
   const outputDir = fs.mkdtempSync(os.tmpdir() + '/lh-smoke-cdt-runner-');
   const chromeFlags = [

lighthouse-core/scripts/build-devtools.sh

@@ -12,7 +12,7 @@ set -euo pipefail
 # 2) Rolls to local devtools repo. By default, this is the temporary checkout in .tmp
 # 3) Builds devtools frontend with new Lighthouse roll
 # 
-# Run `bash lighthouse-core/test/chromium-web-tests/setup.sh` first to update the temporary devtools checkout.
+# Run `bash lighthouse-core/test/devtools-tests/setup.sh` first to update the temporary devtools checkout.
 # Specify `$DEVTOOLS_PATH` to use a different devtools repo.
 
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
@@ -35,15 +35,12 @@ fi
 if ! which gn ; then
   # If the contributor doesn't have a separate depot tools in their path, use the tmp copy.
   DEPOT_TOOLS_PATH="$TEST_DIR/depot-tools"
-  BLINK_TOOLS_PATH="$TEST_DIR/blink_tools"
   export PATH=$DEPOT_TOOLS_PATH:$PATH
-  # Add typ to python path. The regular method assumes there is a Chromium checkout.
-  export PYTHONPATH="${PYTHONPATH:-}:$BLINK_TOOLS_PATH/latest/third_party/typ"
 fi
 
 yarn devtools "$DEVTOOLS_PATH"
 
 cd "$DEVTOOLS_PATH"
-gn gen out/Default
+gn gen out/Default --args='devtools_dcheck_always_on=true is_debug=false'
 gclient sync
 autoninja -C out/Default

lighthouse-core/scripts/open-devtools.sh

@@ -23,7 +23,7 @@ fi
 
 # If using the default .tmp devtools checkout, make sure it's up to date first.
 if [ -z "${DEVTOOLS_PATH:-}" ]; then
-  source "$LH_ROOT/lighthouse-core/test/chromium-web-tests/setup.sh"
+  source "$LH_ROOT/lighthouse-core/test/devtools-tests/setup.sh"
 fi
 
 echo "CHROME_PATH: $CHROME_PATH"

lighthouse-core/scripts/pptr-run-devtools.js

@@ -14,7 +14,7 @@
  *    yarn run-devtools --chrome-flags=--custom-devtools-frontend=file://$HOME/src/devtools/devtools-frontend/out/Default/gen/front_end
  *
  * Or with the DevTools in .tmp:
- *   bash lighthouse-core/test/chromium-web-tests/setup.sh
+ *   bash lighthouse-core/test/devtools-tests/setup.sh
  *   yarn run-devtools --chrome-flags=--custom-devtools-frontend=file://$PWD/.tmp/chromium-web-tests/devtools/devtools-frontend/out/Default/gen/front_end
  *
  * URL list file: yarn run-devtools < path/to/urls.txt

lighthouse-core/scripts/roll-to-devtools.sh

@@ -73,14 +73,6 @@ fe_locales_dir="$fe_lh_dir/locales"
 rsync -avh "$lh_locales_dir" "$fe_locales_dir" --exclude="*.ctc.json" --delete
 echo -e "$check Locale JSON files copied."
 
-# copy webtests
-lh_webtests_dir="third-party/chromium-webtests/webtests/http/tests/devtools/lighthouse/"
-fe_webtests_dir="$dt_dir/test/webtests/http/tests/devtools/lighthouse"
-rsync -avh "$lh_webtests_dir" "$fe_webtests_dir" --exclude="OWNERS" --delete
-lh_webtests_exp_dir="third-party/chromium-webtests/webtests/platform/generic/http/tests/devtools/lighthouse/"
-fe_webtests_exp_dir="$dt_dir/test/webtests/platform/generic/http/tests/devtools/lighthouse"
-rsync -avh "$lh_webtests_exp_dir" "$fe_webtests_exp_dir" --exclude="OWNERS" --delete
-
 # copy e2e tests
 lh_e2e_dir="third-party/devtools-tests/e2e/lighthouse/"
 fe_e2e_dir="$dt_dir/test/e2e/lighthouse"
@@ -90,5 +82,5 @@ fe_e2e_res_dir="$dt_dir/test/e2e/resources/lighthouse"
 rsync -avh "$lh_e2e_res_dir" "$fe_e2e_res_dir" --exclude="OWNERS" --delete
 
 echo ""
-echo "Done. To run the webtests: "
+echo "Done. To run the e2e tests: "
 echo "    DEVTOOLS_PATH=\"$dt_dir\" yarn test-devtools"