Add next experimental-test command (#64352)

This PR adds a new `experimental-test` command to Next.js CLI.

It has 3 main functions:
- auto installing missing playwright dependencies
- auto generating missing playwright config
- running tests via `playwright test`

I'm currently working on sharing a public RFC that will have more
information. I will link that here when its available.

Closes NEXT-3076 NEXT-3032

---------

Co-authored-by: samcx <sam@vercel.com>

packages/next/src/bin/next.ts

@@ -10,6 +10,7 @@ import { bold, cyan, italic } from '../lib/picocolors'
 import { formatCliHelpOutput } from '../lib/format-cli-help-output'
 import { NON_STANDARD_NODE_ENV } from '../lib/constants'
 import { myParseInt } from '../server/lib/utils'
+import { SUPPORTED_TEST_RUNNERS_LIST } from '../cli/next-test.js'
 
 if (
   semver.lt(process.versions.node, process.env.__NEXT_REQUIRED_NODE_VERSION!)
@@ -346,6 +347,36 @@ program
       mod.nextTelemetry(options, arg)
     )
   )
-  .usage('[options]')
+
+program
+  .command('experimental-test')
+  .description(
+    `Execute \`next/experimental/testmode\` tests using a specified test runner. The test runner defaults to 'playwright' if the \`experimental.defaultTestRunner\` configuration option or the \`--test-runner\` option are not set.`
+  )
+  .argument(
+    '[directory]',
+    `A Next.js project directory to execute the test runner on. ${italic(
+      'If no directory is provided, the current directory will be used.'
+    )}`
+  )
+  .argument(
+    '[test-runner-args...]',
+    'Any additional arguments or options to pass down to the test runner `test` command.'
+  )
+  .option(
+    '--test-runner [test-runner]',
+    `Any supported test runner. Options: ${bold(
+      SUPPORTED_TEST_RUNNERS_LIST.join(', ')
+    )}. ${italic(
+      "If no test runner is provided, the Next.js config option `experimental.defaultTestRunner`, or 'playwright' will be used."
+    )}`
+  )
+  .allowUnknownOption()
+  .action((directory, testRunnerArgs, options) => {
+    return import('../cli/next-test.js').then((mod) => {
+      mod.nextTest(directory, testRunnerArgs, options)
+    })
+  })
+  .usage('[directory] [options]')
 
 program.parse(process.argv)

packages/next/src/cli/next-test.ts

@@ -0,0 +1,195 @@
+import { writeFileSync } from 'fs'
+import { getProjectDir } from '../lib/get-project-dir'
+import { printAndExit } from '../server/lib/utils'
+import loadConfig from '../server/config'
+import { PHASE_PRODUCTION_BUILD } from '../shared/lib/constants'
+import {
+  hasNecessaryDependencies,
+  type MissingDependency,
+} from '../lib/has-necessary-dependencies'
+import { installDependencies } from '../lib/install-dependencies'
+import type { NextConfigComplete } from '../server/config-shared'
+import findUp from 'next/dist/compiled/find-up'
+import { findPagesDir } from '../lib/find-pages-dir'
+import { verifyTypeScriptSetup } from '../lib/verify-typescript-setup'
+import path from 'path'
+import spawn from 'next/dist/compiled/cross-spawn'
+
+export interface NextTestOptions {
+  testRunner?: string
+}
+
+export const SUPPORTED_TEST_RUNNERS_LIST = ['playwright'] as const
+export type SupportedTestRunners = (typeof SUPPORTED_TEST_RUNNERS_LIST)[number]
+
+const requiredPackagesByTestRunner: {
+  [k in SupportedTestRunners]: MissingDependency[]
+} = {
+  playwright: [
+    { file: 'playwright', pkg: '@playwright/test', exportsRestrict: false },
+  ],
+}
+
+export async function nextTest(
+  directory?: string,
+  testRunnerArgs: string[] = [],
+  options: NextTestOptions = {}
+) {
+  // The following mess is in order to support an existing Next.js CLI pattern of optionally, passing a project `directory` as the first argument to execute the command on.
+  // This is problematic for `next test` because as a wrapper around a test runner's `test` command, it needs to pass through any additional arguments and options.
+  // Thus, `directory` could either be a valid Next.js project directory (that the user intends to run `next test` on), or it is the first argument for the test runner.
+  // Unfortunately, since many test runners support passing a path (to a test file or directory containing test files), we must check if `directory` is both a valid path and a valid Next.js project.
+
+  let baseDir, nextConfig
+
+  try {
+    // if directory is `undefined` or a valid path this will succeed.
+    baseDir = getProjectDir(directory, false)
+  } catch (err) {
+    // if that failed, then `directory` is not a valid path, so it must have meant to be the first item for `testRunnerArgs`
+    // @ts-expect-error directory is a string here since `getProjectDir` will succeed if its undefined
+    testRunnerArgs.unshift(directory)
+    // intentionally set baseDir to the resolved '.' path
+    baseDir = getProjectDir()
+  }
+
+  try {
+    // but, `baseDir` might not be a Next.js project directory, it could be a path-like argument for the test runner (i.e. `playwright test test/foo.spec.js`)
+    // if this succeeds, it means that `baseDir` is a Next.js project directory
+    nextConfig = await loadConfig(PHASE_PRODUCTION_BUILD, baseDir)
+  } catch (err) {
+    // if it doesn't, then most likely `baseDir` is not a Next.js project directory
+    // @ts-expect-error directory is a string here since `getProjectDir` will succeed if its undefined
+    testRunnerArgs.unshift(directory)
+    // intentionally set baseDir to the resolved '.' path
+    baseDir = getProjectDir()
+    nextConfig = await loadConfig(PHASE_PRODUCTION_BUILD, baseDir) // let this error bubble up if the `basePath` is still not a valid Next.js project
+  }
+
+  // set the test runner. priority is CLI option > next config > default 'playwright'
+  const configuredTestRunner =
+    options?.testRunner ?? // --test-runner='foo'
+    nextConfig.experimental.defaultTestRunner ?? // { experimental: { defaultTestRunner: 'foo' }}
+    'playwright'
+
+  if (!nextConfig.experimental.testProxy) {
+    return printAndExit(
+      `\`next experimental-test\` requires the \`experimental.testProxy: true\` configuration option.`
+    )
+  }
+
+  // execute test runner specific function
+  switch (configuredTestRunner) {
+    case 'playwright':
+      return runPlaywright(baseDir, nextConfig, testRunnerArgs)
+    default:
+      return printAndExit(
+        `Test runner ${configuredTestRunner} is not supported.`
+      )
+  }
+}
+
+async function checkRequiredDeps(
+  baseDir: string,
+  testRunner: SupportedTestRunners
+) {
+  const deps = await hasNecessaryDependencies(
+    baseDir,
+    requiredPackagesByTestRunner[testRunner]
+  )
+  if (deps.missing.length > 0) {
+    await installDependencies(baseDir, deps.missing, true)
+
+    const playwright = spawn(
+      path.join(baseDir, 'node_modules', '.bin', 'playwright'),
+      ['install'],
+      {
+        cwd: baseDir,
+        shell: false,
+        stdio: 'inherit',
+        env: {
+          ...process.env,
+        },
+      }
+    )
+
+    return new Promise((resolve, reject) => {
+      playwright.on('close', (c) => resolve(c))
+      playwright.on('error', (err) => reject(err))
+    })
+  }
+}
+
+async function runPlaywright(
+  baseDir: string,
+  nextConfig: NextConfigComplete,
+  testRunnerArgs: string[]
+) {
+  await checkRequiredDeps(baseDir, 'playwright')
+
+  const playwrightConfigFile = await findUp(
+    ['playwright.config.js', 'playwright.config.ts'],
+    {
+      cwd: baseDir,
+    }
+  )
+
+  if (!playwrightConfigFile) {
+    const { pagesDir, appDir } = findPagesDir(baseDir)
+
+    const { version: typeScriptVersion } = await verifyTypeScriptSetup({
+      dir: baseDir,
+      distDir: nextConfig.distDir,
+      intentDirs: [pagesDir, appDir].filter(Boolean) as string[],
+      typeCheckPreflight: false,
+      tsconfigPath: nextConfig.typescript.tsconfigPath,
+      disableStaticImages: nextConfig.images.disableStaticImages,
+      hasAppDir: !!appDir,
+      hasPagesDir: !!pagesDir,
+    })
+
+    const isUsingTypeScript = !!typeScriptVersion
+
+    const playwrightConfigFilename = isUsingTypeScript
+      ? 'playwright.config.ts'
+      : 'playwright.config.js'
+
+    writeFileSync(
+      path.join(baseDir, playwrightConfigFilename),
+      defaultPlaywrightConfig(isUsingTypeScript)
+    )
+
+    return printAndExit(
+      `Successfully generated ${playwrightConfigFilename}. Create your first test and then run \`next experimental-test\`.`,
+      0
+    )
+  } else {
+    const playwright = spawn(
+      path.join(baseDir, 'node_modules', '.bin', 'playwright'),
+      ['test', ...testRunnerArgs],
+      {
+        cwd: baseDir,
+        shell: false,
+        stdio: 'inherit',
+        env: {
+          ...process.env,
+        },
+      }
+    )
+    return new Promise((resolve, reject) => {
+      playwright.on('close', (c) => resolve(c))
+      playwright.on('error', (err) => reject(err))
+    })
+  }
+}
+
+function defaultPlaywrightConfig(typescript: boolean) {
+  const comment = `/*
+ * Specify any additional Playwright config options here.
+ * They will be merged with Next.js' default Playwright config.
+ * You can access the default config by importing \`defaultPlaywrightConfig\` from \`'next/experimental/testmode/playwright'\`.
+ */`
+  return typescript
+    ? `import { defineConfig } from 'next/experimental/testmode/playwright';\n\n${comment}\nexport default defineConfig({});`
+    : `const { defineConfig } = require('next/experimental/testmode/playwright');\n\n${comment}\nmodule.exports = defineConfig({});`
+}

packages/next/src/experimental/testmode/playwright/default-config.ts

@@ -0,0 +1,43 @@
+import { devices, type PlaywrightTestConfig } from '@playwright/test'
+import type { NextOptionsConfig } from './next-options'
+
+/**
+ * This is the default configuration generated by Playwright as of v1.43.0 with some modifications.
+ *
+ * - the `testMatch` property is configured to match all `*.spec.js` or `*.spec.ts` files within the `app` and `pages` directories
+ * - the `use` property is configured with a baseURL matching the expected dev server endpoint (http://127.0.0.1:3000)
+ * - the `webserver` property is configured to run `next dev`.
+ */
+export const defaultPlaywrightConfig: PlaywrightTestConfig<NextOptionsConfig> =
+  {
+    testMatch: '{app,pages}/**/*.spec.{t,j}s',
+    fullyParallel: true,
+    forbidOnly: process.env.CI === 'true',
+    retries: process.env.CI === 'true' ? 2 : 0,
+    reporter: [['list'], ['html', { open: 'never' }]],
+    use: {
+      baseURL: 'http://127.0.0.1:3000',
+      trace: 'on-first-retry',
+    },
+    projects: [
+      {
+        name: 'chromium',
+        use: { ...devices['Desktop Chrome'] },
+      },
+
+      {
+        name: 'firefox',
+        use: { ...devices['Desktop Firefox'] },
+      },
+
+      {
+        name: 'webkit',
+        use: { ...devices['Desktop Safari'] },
+      },
+    ],
+    webServer: {
+      command: process.env.CI === 'true' ? 'next start' : 'next dev',
+      url: 'http://127.0.0.1:3000',
+      reuseExistingServer: process.env.CI !== 'true',
+    },
+  }

packages/next/src/experimental/testmode/playwright/index.ts

@@ -1,30 +1,33 @@
 // eslint-disable-next-line import/no-extraneous-dependencies
 import * as base from '@playwright/test'
 import type { NextFixture } from './next-fixture'
-import type { NextOptions } from './next-options'
+import type { NextOptions, NextOptionsConfig } from './next-options'
 import type { NextWorkerFixture } from './next-worker-fixture'
 import { applyNextWorkerFixture } from './next-worker-fixture'
 import { applyNextFixture } from './next-fixture'
+import { defaultPlaywrightConfig } from './default-config'
+
+export { defaultPlaywrightConfig }
 
 // eslint-disable-next-line import/no-extraneous-dependencies
 export * from '@playwright/test'
 
-export type { NextFixture, NextOptions }
-export type { FetchHandlerResult } from '../proxy'
-
-export interface NextOptionsConfig {
-  nextOptions?: NextOptions
-}
-
+// Export this second so it overrides the one from `@playwright/test`
 export function defineConfig<T extends NextOptionsConfig, W>(
   config: base.PlaywrightTestConfig<T, W>
 ): base.PlaywrightTestConfig<T, W>
 export function defineConfig<T extends NextOptionsConfig = NextOptionsConfig>(
   config: base.PlaywrightTestConfig<T>
 ): base.PlaywrightTestConfig<T> {
-  return base.defineConfig<T>(config)
+  return base.defineConfig<T>(
+    defaultPlaywrightConfig as base.PlaywrightTestConfig<T>,
+    config
+  )
 }
 
+export type { NextFixture, NextOptions }
+export type { FetchHandlerResult } from '../proxy'
+
 export const test = base.test.extend<
   { next: NextFixture; nextOptions: NextOptions },
   { _nextWorker: NextWorkerFixture }

packages/next/src/experimental/testmode/playwright/next-options.ts

@@ -1,3 +1,7 @@
 export interface NextOptions {
   fetchLoopback?: boolean
 }
+
+export interface NextOptionsConfig {
+  nextOptions?: NextOptions
+}

packages/next/src/lib/get-project-dir.ts

@@ -1,11 +1,12 @@
 import path from 'path'
-import { error, warn } from '../build/output/log'
+import { warn } from '../build/output/log'
 import { detectTypo } from './detect-typo'
 import { realpathSync } from './realpath'
+import { printAndExit } from '../server/lib/utils'
 
-export function getProjectDir(dir?: string) {
+export function getProjectDir(dir?: string, exitOnEnoent = true) {
+  const resolvedDir = path.resolve(dir || '.')
   try {
-    const resolvedDir = path.resolve(dir || '.')
     const realDir = realpathSync(resolvedDir)
 
     if (
@@ -19,7 +20,7 @@ export function getProjectDir(dir?: string) {
 
     return realDir
   } catch (err: any) {
-    if (err.code === 'ENOENT') {
+    if (err.code === 'ENOENT' && exitOnEnoent) {
       if (typeof dir === 'string') {
         const detectedTypo = detectTypo(dir, [
           'build',
@@ -28,22 +29,19 @@ export function getProjectDir(dir?: string) {
           'lint',
           'start',
           'telemetry',
+          'experimental-test',
         ])
 
         if (detectedTypo) {
-          error(
+          return printAndExit(
             `"next ${dir}" does not exist. Did you mean "next ${detectedTypo}"?`
           )
-          process.exit(1)
         }
       }
 
-      error(
-        `Invalid project directory provided, no such directory: ${path.resolve(
-          dir || '.'
-        )}`
+      return printAndExit(
+        `Invalid project directory provided, no such directory: ${resolvedDir}`
       )
-      process.exit(1)
     }
     throw err
   }