test(turbopack): run next.js integration tests with denylist (vercel#…

…50172)

<!-- Thanks for opening a PR! Your contribution is much appreciated.
To make sure your PR is handled as smoothly as possible we request that
you follow the checklist sections below.
Choose the right checklist for the change(s) that you're making:

## For Contributors

### Improving Documentation or adding/fixing Examples

- The "examples guidelines" are followed from our contributing doc
https://github.com/vercel/next.js/blob/canary/contributing/examples/adding-examples.md
- Make sure the linting passes by running `pnpm build && pnpm lint`. See
https://github.com/vercel/next.js/blob/canary/contributing/repository/linting.md

### Fixing a bug

- Related issues linked using `fixes #number`
- Tests added. See:
https://github.com/vercel/next.js/blob/canary/contributing/core/testing.md#writing-tests-for-nextjs
- Errors have a helpful link attached, see
https://github.com/vercel/next.js/blob/canary/contributing.md

### Adding a feature

- Implements an existing feature request or RFC. Make sure the feature
request has been accepted for implementation before opening a PR. (A
discussion must be opened, see
https://github.com/vercel/next.js/discussions/new?category=ideas)
- Related issues/discussions are linked using `fixes #number`
- e2e tests added
(https://github.com/vercel/next.js/blob/canary/contributing/core/testing.md#writing-tests-for-nextjs
- Documentation added
- Telemetry added. In case of a feature if it's used or not.
- Errors have a helpful link attached, see
https://github.com/vercel/next.js/blob/canary/contributing.md



## For Maintainers

- Minimal description (aim for explaining to someone not on the team to
understand the PR)
- When linking to a Slack thread, you might want to share details of the
conclusion
- Link both the Linear (Fixes NEXT-xxx) and the GitHub issues
- Add review comments if necessary to explain to the reviewer the logic
behind a change



### Why?

### How?

Closes NEXT-
Fixes #

-->

### What?

WEB-1086.

This PR flips the way running next.js integration test with Turbopack,
instead of using allowlist to run selectively enabled test now uses
denylists. The immediate effect is now any new test being added (as a
new file) will be executed with Turbopack by default, and should be
manually excluded if it fails.

Still, the number of tests we run remains same.

.github/workflows/build_test_deploy.yml

@@ -812,9 +812,8 @@ jobs:
       - run: xvfb-run node run-tests.js test/integration/with-electron/test/index.test.js
         if: ${{needs.build.outputs.docsChange == 'nope'}}
 
-  # A job to run sets of tests with turbopack enabled. These tests are considered as `stable`,
-  # that running with turbopack should always pass.
-  testTurbopack:
+  # A job to run sets of devlopment tests with turbopack enabled.
+  testDevTurbopack:
     name: Test Development (Turbopack)
     runs-on: ubuntu-latest
     needs: [build, build-native-test]
@@ -832,6 +831,10 @@ jobs:
 
     strategy:
       fail-fast: false
+      matrix:
+        node: [18]
+        # Currently turbopack tests does not utilize groups yet
+        group: [1]
     steps:
       - run: echo "${{needs.build.outputs.docsChange}}"
 
@@ -853,10 +856,119 @@ jobs:
           name: next-swc-test-binary
           path: packages/next-swc/native
 
-      - run: docker run --rm -v $(pwd):/work mcr.microsoft.com/playwright:v1.28.1-jammy /bin/bash -c "cd /work && NODE_VERSION=${{ env.NODE_LTS_VERSION }} ./scripts/setup-node.sh && node -v && npm i -g pnpm@${PNPM_VERSION} > /dev/null && NEXT_EXTERNAL_TESTS_FILTERS=${NEXT_EXTERNAL_TESTS_FILTERS} __INTERNAL_NEXT_DEV_TEST_TURBO_DEV=TRUE __INTERNAL_CUSTOM_TURBOPACK_BINDINGS=${NEXT_BINDINGS_BIN} __INTERNAL_NEXT_DEV_TEST_TURBO_GLOB_MATCH=${NEXT_DEV_TEST_GLOB} NEXT_E2E_TEST_TIMEOUT=240000 NEXT_TEST_JOB=1 NEXT_TEST_MODE=dev TEST_TIMINGS_TOKEN=${{ secrets.TEST_TIMINGS_TOKEN }} xvfb-run node run-tests.js --type development --timings -c 1 >> /proc/1/fd/1"
+      - run: docker run --rm -v $(pwd):/work mcr.microsoft.com/playwright:v1.28.1-jammy /bin/bash -c "cd /work && NODE_VERSION=${{ env.NODE_LTS_VERSION }} ./scripts/setup-node.sh && node -v && npm i -g pnpm@${PNPM_VERSION} > /dev/null && NEXT_EXTERNAL_TESTS_FILTERS=${NEXT_EXTERNAL_TESTS_FILTERS} __INTERNAL_NEXT_DEV_TEST_TURBO_DEV=TRUE __INTERNAL_CUSTOM_TURBOPACK_BINDINGS=${NEXT_BINDINGS_BIN} __INTERNAL_NEXT_DEV_TEST_TURBO_GLOB_MATCH=${NEXT_DEV_TEST_GLOB} NEXT_E2E_TEST_TIMEOUT=240000 NEXT_TEST_JOB=1 NEXT_TEST_MODE=dev TEST_TIMINGS_TOKEN=${{ secrets.TEST_TIMINGS_TOKEN }} xvfb-run node run-tests.js --type development --timings >> /proc/1/fd/1"
         name: Run test/development
         if: ${{needs.build.outputs.docsChange == 'nope'}}
 
+  testDevE2ETurbopack:
+    name: Test Development (E2E) (Turbopack)
+    runs-on: ubuntu-latest
+    needs: [build, build-native-test]
+    timeout-minutes: 35
+    env:
+      NEXT_TELEMETRY_DISABLED: 1
+      TEST_TIMINGS_TOKEN: ${{ secrets.TEST_TIMINGS_TOKEN }}
+      # Enabling backtrace will makes snapshot tests fail
+      RUST_BACKTRACE: 0
+      # Path to the custom next-swc bindings.
+      NEXT_BINDINGS_BIN: /work/packages/next-swc/native/next-swc.linux-x64-gnu.node
+      # Glob pattern to run specific tests with --turbo.
+      NEXT_DEV_TEST_GLOB: '*'
+      NEXT_EXTERNAL_TESTS_FILTERS: /work/packages/next-swc/crates/next-dev-tests/tests-manifest.js
+    strategy:
+      fail-fast: false
+      matrix:
+        node: [18]
+        # Currently turbopack tests does not utilize groups yet
+        group: [1]
+    steps:
+      - run: echo "${{needs.build.outputs.docsChange}}"
+
+      # https://github.com/actions/virtual-environments/issues/1187
+      - name: tune linux network
+        run: sudo ethtool -K eth0 tx off rx off
+
+      - uses: actions/cache@v3
+        timeout-minutes: 5
+        if: ${{needs.build.outputs.docsChange == 'nope'}}
+        id: restore-build
+        with:
+          path: ./*
+          key: ${{ github.sha }}-${{ github.run_number }}
+
+      - uses: actions/download-artifact@v3
+        if: ${{needs.build.outputs.docsChange == 'nope'}}
+        with:
+          name: next-swc-test-binary
+          path: packages/next-swc/native
+
+      - run: docker run --rm -v $(pwd):/work mcr.microsoft.com/playwright:v1.28.1-jammy /bin/bash -c "cd /work && curl -s https://install-node.vercel.app/v${{ matrix.node }} | FORCE=1 bash && node -v && npm i -g pnpm@${PNPM_VERSION} > /dev/null && NEXT_EXTERNAL_TESTS_FILTERS=${NEXT_EXTERNAL_TESTS_FILTERS} __INTERNAL_NEXT_DEV_TEST_TURBO_DEV=TRUE __INTERNAL_CUSTOM_TURBOPACK_BINDINGS=${NEXT_BINDINGS_BIN} __INTERNAL_NEXT_DEV_TEST_TURBO_GLOB_MATCH=${NEXT_DEV_TEST_GLOB} NEXT_E2E_TEST_TIMEOUT=240000 NEXT_TEST_JOB=1 NEXT_TEST_MODE=dev TEST_TIMINGS_TOKEN=${{ secrets.TEST_TIMINGS_TOKEN }} xvfb-run node run-tests.js --type e2e --timings >> /proc/1/fd/1"
+        name: Run test/e2e (dev)
+        if: ${{needs.build.outputs.docsChange == 'nope'}}
+
+      - name: Upload test trace
+        if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: test-trace
+          if-no-files-found: ignore
+          retention-days: 2
+          path: |
+            test/traces
+
+  testIntegrationTurbopack:
+    name: Test Integration (Turbopack)
+    runs-on: ubuntu-latest
+    needs: [build, build-native-test]
+    timeout-minutes: 35
+    env:
+      NEXT_TELEMETRY_DISABLED: 1
+      TEST_TIMINGS_TOKEN: ${{ secrets.TEST_TIMINGS_TOKEN }}
+      # Enabling backtrace will makes snapshot tests fail
+      RUST_BACKTRACE: 0
+      # Path to the custom next-swc bindings.
+      NEXT_BINDINGS_BIN: /work/packages/next-swc/native/next-swc.linux-x64-gnu.node
+      # Glob pattern to run specific tests with --turbo.
+      NEXT_DEV_TEST_GLOB: '*'
+      NEXT_EXTERNAL_TESTS_FILTERS: /work/packages/next-swc/crates/next-dev-tests/tests-manifest.js
+    strategy:
+      fail-fast: false
+      matrix:
+        group: [1]
+    steps:
+      - run: echo "${{needs.build.outputs.docsChange}}"
+
+      # https://github.com/actions/virtual-environments/issues/1187
+      - name: tune linux network
+        run: sudo ethtool -K eth0 tx off rx off
+
+      - uses: actions/cache@v3
+        timeout-minutes: 5
+        if: ${{needs.build.outputs.docsChange == 'nope'}}
+        id: restore-build
+        with:
+          path: ./*
+          key: ${{ github.sha }}-${{ github.run_number }}
+
+      - uses: actions/download-artifact@v3
+        if: ${{needs.build.outputs.docsChange == 'nope'}}
+        with:
+          name: next-swc-test-binary
+          path: packages/next-swc/native
+
+      - run: docker run --rm -v $(pwd):/work mcr.microsoft.com/playwright:v1.28.1-jammy /bin/bash -c "cd /work && NODE_VERSION=${{ env.NODE_LTS_VERSION }} ./scripts/setup-node.sh && npm i -g pnpm@${PNPM_VERSION} > /dev/null && NEXT_EXTERNAL_TESTS_FILTERS=${NEXT_EXTERNAL_TESTS_FILTERS} __INTERNAL_NEXT_DEV_TEST_TURBO_DEV=TRUE __INTERNAL_CUSTOM_TURBOPACK_BINDINGS=${NEXT_BINDINGS_BIN} __INTERNAL_NEXT_DEV_TEST_TURBO_GLOB_MATCH=${NEXT_DEV_TEST_GLOB} NEXT_E2E_TEST_TIMEOUT=240000 NEXT_TEST_JOB=1 TEST_TIMINGS_TOKEN=${{ secrets.TEST_TIMINGS_TOKEN }} xvfb-run node run-tests.js --timings >> /proc/1/fd/1"
+        if: ${{needs.build.outputs.docsChange == 'nope'}}
+
+      - name: Upload test trace
+        if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: test-trace
+          if-no-files-found: ignore
+          retention-days: 2
+          path: |
+            test/traces
+
   testsPass:
     name: thank you, next
     runs-on: ubuntu-latest

packages/next-swc/crates/next-dev-tests/sync-tests-manifest.js

@@ -8,10 +8,11 @@ const path = require('path')
 
 const generateManifest = (enabledTests, disabledTests) => `
 // Tests that are currently enabled with Turbopack in CI.
-// Add new test when Turbopack updates to fix / implement a feature.
+// This list is not actively used, more of high level pictures of what tests are enabled.
 const enabledTests = ${enabledTests}
 
 // Tests that are currently disabled with Turbopack in CI.
+// Any tests not listed in here are assumed to be enabled.
 const disabledTests = ${disabledTests}
 
 module.exports = {
@@ -30,7 +31,7 @@ const main = async () => {
     enabledTests = manifest.enabledTests
     disabledTests = manifest.disabledTests
   } else {
-    throw new Error('a')
+    throw new Error('Manifest should exists')
   }
 
   // Collect all test files

packages/next-swc/crates/next-dev-tests/tests-manifest.js

@@ -1,5 +1,5 @@
 // Tests that are currently enabled with Turbopack in CI.
-// Add new test when Turbopack updates to fix / implement a feature.
+// This list is not actively used, more of high level pictures of what tests are enabled.
 const enabledTests = [
   'test/development/acceptance-app/dynamic-error.test.ts',
   'test/development/basic/legacy-decorators.test.ts',
@@ -9,6 +9,7 @@ const enabledTests = [
 ]
 
 // Tests that are currently disabled with Turbopack in CI.
+// Any tests not listed in here are assumed to be enabled.
 const disabledTests = [
   'test/development/acceptance-app/app-hmr-changes.test.ts',
   'test/development/acceptance-app/component-stack.test.ts',
@@ -76,6 +77,7 @@ const disabledTests = [
   'test/e2e/404-page-router/index.test.ts',
   'test/e2e/app-dir-legacy-edge-runtime-config/index.test.ts',
   'test/e2e/app-dir/_allow-underscored-root-directory/_allow-underscored-root-directory.test.ts',
+  'test/e2e/app-dir/actions/app-action-export.test.ts',
   'test/e2e/app-dir/actions/app-action.test.ts',
   'test/e2e/app-dir/app-a11y/index.test.ts',
   'test/e2e/app-dir/app-alias/app-alias.test.ts',
@@ -108,6 +110,7 @@ const disabledTests = [
   'test/e2e/app-dir/dynamic-href/dynamic-href.test.ts',
   'test/e2e/app-dir/dynamic/dynamic.test.ts',
   'test/e2e/app-dir/edge-runtime-node-compatibility/edge-runtime-node-compatibility.test.ts',
+  'test/e2e/app-dir/error-boundary-and-not-found-linking/error-boundary-and-not-found-linking.test.ts',
   'test/e2e/app-dir/front-redirect-issue/front-redirect-issue.test.ts',
   'test/e2e/app-dir/global-error/global-error.test.ts',
   'test/e2e/app-dir/hooks/hooks.test.ts',
@@ -672,6 +675,7 @@ const disabledTests = [
   'test/unit/esm-interpolate/esm-interpolate.test.tsx',
   'test/unit/find-config.test.ts',
   'test/unit/find-page-file.test.ts',
+  'test/unit/get-module-build-info.test.ts',
   'test/unit/get-node-options-without-inspect.test.ts',
   'test/unit/get-page-static-infos.test.ts',
   'test/unit/get-project-dir.test.ts',

run-tests.js

@@ -12,11 +12,11 @@ const { createNextInstall } = require('./test/lib/create-next-install')
 const glob = promisify(_glob)
 const exec = promisify(execOrig)
 
-// Try to read an external array-based json to filter tests to be executed.
+// Try to read an external array-based json to filter tests to be allowed / or disallowed.
 // If process.argv contains a test to be executed, this'll append it to the list.
 const externalTestsFilterLists = process.env.NEXT_EXTERNAL_TESTS_FILTERS
-  ? require(process.env.NEXT_EXTERNAL_TESTS_FILTERS).enabledTests
-  : []
+  ? require(process.env.NEXT_EXTERNAL_TESTS_FILTERS)
+  : { enabledTests: [], disabledTests: [] }
 const timings = []
 const DEFAULT_NUM_RETRIES = os.platform() === 'win32' ? 2 : 1
 const DEFAULT_CONCURRENCY = 2
@@ -143,9 +143,7 @@ async function main() {
 
   console.log('Running tests with concurrency:', concurrency)
 
-  let tests = process.argv
-    .filter((arg) => arg.match(/\.test\.(js|ts|tsx)/))
-    .concat(externalTestsFilterLists)
+  let tests = process.argv.filter((arg) => arg.match(/\.test\.(js|ts|tsx)/))
   let prevTimings
 
   if (tests.length === 0) {
@@ -190,6 +188,17 @@ async function main() {
     }
   }
 
+  // If there are external manifest contains list of tests, apply it to the test lists.
+  // Specifically, we filters out `disabledTests` from named export of the manifest.
+  if (externalTestsFilterLists?.disabledTests.length > 0) {
+    tests = tests.filter(
+      (test) =>
+        !externalTestsFilterLists.disabledTests.some((disabled) =>
+          disabled.includes(test)
+        )
+    )
+  }
+
   let testNames = [
     ...new Set(
       tests.map((f) => {