Merge branch 'main' into elef/next-13-e2e

.github/actions/init/action.yml

@@ -38,6 +38,10 @@ inputs:
     description: 'The registry the packages will be published to'
     required: false
     default: ''
+  verbose:
+    description: 'Enable verbose output'
+    required: false
+    default: 'false'
 
 runs:
   using: 'composite'
@@ -54,6 +58,7 @@ runs:
         SUMMARIZE: ${{ inputs.turbo-summarize }}
         TEAM: ${{ inputs.turbo-team }}
         TOKEN: ${{ inputs.turbo-token }}
+        VERBOSE: ${{ inputs.verbose }}
       with:
         script: |
           const os = require('os')
@@ -62,15 +67,24 @@ runs:
               ? os.availableParallelism()
               : os.cpus().length;
 
-          const { ENABLED, CACHE_DIR, SIGNATURE, REMOTE_ONLY, SUMMARIZE, TEAM, TOKEN } = process.env
+          const { ENABLED, CACHE_DIR, SIGNATURE, REMOTE_ONLY, SUMMARIZE, TEAM, TOKEN, VERBOSE } = process.env
+
+          const ARGS = [
+            `--cache-dir=${CACHE_DIR}`,
+            `--concurrency=${cpus}`,
+            `--summarize=${SUMMARIZE}`,
+          ]
+
+          if (VERBOSE === 'true') {
+            ARGS.push('--verbosity=2')
+            ARGS.push('--log-order=stream')
+            ARGS.push('--output-logs=full')
+          } else {
+            ARGS.push('--output-logs=new-only')
+          }
 
           core.exportVariable('TURBO_ARGS',
-            [
-              '--output-logs=new-only',
-              `--cache-dir=${CACHE_DIR}`,
-              `--concurrency=${cpus}`,
-              `--summarize=${SUMMARIZE}`,
-            ].join(' ')
+            ARGS.join(' ')
           )
 
           if (ENABLED === 'true') {

.github/workflows/ci.yml

@@ -129,7 +129,7 @@ jobs:
 
     strategy:
       matrix:
-        test-name: ['generic', 'express', 'quickstart', 'ap-flows']
+        test-name: [ 'generic', 'nextjs', 'express', 'quickstart', 'ap-flows', 'elements' ]
         test-project: ['chrome']
         include:
           - test-name: 'nextjs'
@@ -159,7 +159,7 @@ jobs:
         uses: ./.github/actions/verdaccio
         with:
           publish-cmd: |
-            if [ "$(npm config get registry)" = "https://registry.npmjs.org/" ]; then echo 'Error: Using default registry' && exit 1; else npx turbo build $TURBO_ARGS --filter=!elements --only && npx changeset publish --no-git-tag; fi
+            if [ "$(npm config get registry)" = "https://registry.npmjs.org/" ]; then echo 'Error: Using default registry' && exit 1; else npx turbo build $TURBO_ARGS --only && npx changeset publish --no-git-tag; fi
 
       - name: Install @clerk/backend in /integration
         working-directory: ./integration
@@ -170,6 +170,7 @@ jobs:
         run: mkdir clerk-js && cd clerk-js && npm init -y && npm install @clerk/clerk-js
 
       - name: Run Integration Tests
+        id: integration-tests
         run: npx turbo test:integration:${{ matrix.test-name }} $TURBO_ARGS --only -- --project=${{ matrix.test-project }}
         env:
           E2E_NEXTJS_VERSION: ${{ matrix.next-version }}
@@ -178,10 +179,10 @@ jobs:
           INTEGRATION_INSTANCE_KEYS: ${{ secrets.INTEGRATION_INSTANCE_KEYS }}
           MAILSAC_API_KEY: ${{ secrets.MAILSAC_API_KEY }}
 
-      # - name: Upload Integration Report for ${{ matrix.test-name }}
-      #   uses: actions/upload-artifact@v3
-      #   if: always()
-      #   with:
-      #     name: integration-report-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.test-name }}
-      #     path: playwright-report/
-      #     retention-days: 1
+      - name: Upload test-results
+        if: ${{ cancelled() || failure() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: playwright-traces-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.test-name }}
+          path: integration/test-results
+          retention-days: 1

integration/README.md

@@ -74,6 +74,36 @@ npm run test:integration:base -- --ui email.link.test.ts
 > [!TIP]
 > If you want to learn more, read the [Running and debugging tests documentation](https://playwright.dev/docs/running-tests).
 
+### Recipes
+
+Below you can find code snippets for running tests in a specific manner, easily copy/pasteable. They'll allow you to run tests quicker or make them easier to debug.
+
+#### Keep temporary site
+
+During E2E runs a temporary site is created in which the template is copied into. If you want to keep the site around, pass the `CLEANUP` environment variable:
+
+```shell
+CLEANUP=0 npm run test:integration:base
+```
+
+For all available environment variables, check the [`constants.ts`](../integration/constants.ts) file.
+
+#### Quick feedback loop on already running app
+
+You might have a similar setup to this:
+
+- App running on `localhost:3000` (e.g. running the temporary site that was created in a previous run or directly running the app from the `integration/templates` folder)
+- No need to test `clerk-js` changes
+- As such, you don't need to have a server spin up for `clerk-js` and also don't need an app created
+
+Then you can use a combination of environment variables to fast track your tests:
+
+```shell
+E2E_APP_SK=sk_test_xxx E2E_APP_PK=pk_test_xxx E2E_APP_URL=http://localhost:3000 E2E_APP_CLERK_JS=https://xxx.clerk.accounts.dev/npm/@clerk/clerk-js@5/dist/clerk.browser.js npm run test:integration:base -- --ui
+```
+
+You need to replace all environment variables with your specific values/needs, above are just placeholders.
+
 ## Writing your first test
 
 In this step-by-step instruction you'll learn how to create a new integration test. If your test case already fits into an existing file, please add a new `test()` block instead of creating a whole new file.
@@ -281,6 +311,41 @@ If you need to run a test suite inside a different environment (e.g. a different
 
 1. Ensure that your new keys are added to the `INTEGRATION_INSTANCE_KEYS` environment variable inside the repository so that GitHub actions can successfully run.
 
+## Debugging tests
+
+Sometimes tests are passing locally but not in CI 😢 But there are ways to dig into the root cause. Inside the PR with the failing tests apply these changes:
+
+1. Open the [`ci.yml`](../.github/workflows/ci.yml) file
+1. Inside the **Setup** step (of the `integration-tests` job), add `verbose: true` to the arguments. This will enable more verbose logging from Turborepo and ensure that all logs are flushed the moment they appear.
+1. Playwright will record traces of failed tests and they will be uploaded when the E2E step fails or is cancelled. Click on the **Upload test-results** step and download the archive. It contains Playwright traces.
+1. Open https://trace.playwright.dev/ and open your trace file
+
+If these information are not enough, it might be helpful to have a look at the temporary site that was created inside the test run. You'll need to make some adjustments:
+
+1. Open [`applicationConfig.ts`](../integration/models/applicationConfig.ts) and change the `appDirName` to `${name}__debug`. This way the temporary site doesn't have a random hash and date in its name
+1. In the [`ci.yml`](../.github/workflows/ci.yml) workflow file, change the `test-name` matrix of the `integration-tests` job to only include the project you're interested in. For example:
+
+   ```yaml
+   strategy:
+     matrix:
+       # In the original file the test-name includes more in its array
+       test-name: ['nextjs']
+   ```
+
+1. At the bottom of the file, add a new step to upload the temporary site.
+
+   ```yaml
+   - name: Upload app artifacts
+     if: ${{ cancelled() || failure() }}
+     uses: actions/upload-artifact@v4
+     with:
+       name: temp-app-${{ matrix.test-name }}
+       path: /tmp/.temp_integration/long-running--XXX__debug
+       retention-days: 1
+   ```
+
+You need to replace the `XXX` in the `path` with the ID of your long running app. Those IDs are defined in [`longRunningApps.ts`](../integration/presets/longRunningApps.ts), so check which ID is used for your test. Previous runs might also print the name already, look for a log that begins with "[appConfig] Copying template".
+
 ## Reference
 
 > [!TIP]
@@ -510,10 +575,7 @@ The integration suite uses [`presets/envs.ts`](../integration/presets/envs.ts) t
 
 This is why you created the `.keys.json` file in the [initial setup](#initial-setup) step. Those secret and publishable keys are used to create environment configs. Inside GitHub actions these keys are provided through the `INTEGRATION_INSTANCE_KEYS` environment variable.
 
-Currently, we have two Clerk instances configured:
-
-- **with-email-codes**: a single session application with all toggles enabled.
-- **with-email-links**: a single session app with email links enabled and email codes disabled. Useful to test email links flows as the `<SignUp />` component currently does not support switching between email links and email codes.
+They keys defined in `.keys.json.sample` correspond with the Clerk instances in the **Integration testing** organization.
 
 > [!NOTE]
 > The test suite also uses these environment variables to run some tests:

integration/models/applicationConfig.ts

@@ -54,6 +54,9 @@ export const applicationConfig = () => {
       scripts[name] = cmd;
       return self;
     },
+    /**
+     * Adds a dependency to the template's `package.json` file. If the version is undefined, the dependency is not added. If the dependency already exists, the version is overwritten.
+     */
     addDependency: (name: string, version: string | undefined) => {
       if (version) {
         dependencies.set(name, version);

integration/playwright.config.ts

@@ -21,7 +21,7 @@ export const common: PlaywrightTestConfig = {
   workers: process.env.CI ? numAvailableWorkers : '70%',
   reporter: process.env.CI ? 'line' : 'list',
   use: {
-    trace: 'on-first-retry',
+    trace: 'retain-on-failure',
     bypassCSP: true, // We probably need to limit this to specific tests
   },
 } as const;

integration/presets/elements.ts

@@ -0,0 +1,24 @@
+import { constants } from '../constants';
+import { applicationConfig } from '../models/applicationConfig.js';
+import { templates } from '../templates/index.js';
+
+const clerkNextjsLocal = `file:${process.cwd()}/packages/nextjs`;
+const clerkElementsLocal = `file:${process.cwd()}/packages/elements`;
+
+const nextAppRouter = applicationConfig()
+  .setName('elements-next')
+  .useTemplate(templates['elements-next'])
+  .setEnvFormatter('public', key => `NEXT_PUBLIC_${key}`)
+  .addScript('setup', 'npm i')
+  .addScript('dev', 'npm run dev')
+  .addScript('build', 'npm run build')
+  .addScript('serve', 'npm run start')
+  .addDependency('next', constants.E2E_NEXTJS_VERSION)
+  .addDependency('react', constants.E2E_REACT_VERSION)
+  .addDependency('react-dom', constants.E2E_REACT_DOM_VERSION)
+  .addDependency('@clerk/nextjs', constants.E2E_CLERK_VERSION || clerkNextjsLocal)
+  .addDependency('@clerk/elements', constants.E2E_CLERK_VERSION || clerkElementsLocal);
+
+export const elements = {
+  nextAppRouter,
+} as const;

integration/presets/index.ts

@@ -1,3 +1,4 @@
+import { elements } from './elements';
 import { envs } from './envs';
 import { express } from './express';
 import { createLongRunningApps } from './longRunningApps';
@@ -12,4 +13,5 @@ export const appConfigs = {
   next,
   react,
   remix,
+  elements,
 } as const;

integration/presets/longRunningApps.ts

@@ -1,5 +1,6 @@
 import type { LongRunningApplication } from '../models/longRunningApplication';
 import { longRunningApplication } from '../models/longRunningApplication';
+import { elements } from './elements';
 import { envs } from './envs';
 import { express } from './express';
 import { next } from './next';
@@ -20,13 +21,14 @@ export const createLongRunningApps = () => {
     { id: 'next.appRouter.withEmailCodes', config: next.appRouter, env: envs.withEmailCodes },
     { id: 'next.appRouter.withCustomRoles', config: next.appRouter, env: envs.withCustomRoles },
     { id: 'quickstart.next.appRouter', config: next.appRouterQuickstart, env: envs.withEmailCodesQuickstart },
+    { id: 'elements.next.appRouter', config: elements.nextAppRouter, env: envs.withEmailCodes },
   ] as const;
 
   const apps = configs.map(longRunningApplication);
 
   return {
     getByPattern: (patterns: Array<string | (typeof configs)[number]['id']>) => {
-      const res = new Set(...patterns.map(pattern => apps.filter(app => idMatchesPattern(app.id, pattern))));
+      const res = new Set(patterns.map(pattern => apps.filter(app => idMatchesPattern(app.id, pattern))).flat());
       if (!res.size) {
         const availableIds = configs.map(c => `\n- ${c.id}`).join('');
         throw new Error(`Could not find long running app with id ${patterns}. The available ids are: ${availableIds}`);

integration/templates/elements-next/.eslintrc.js

@@ -0,0 +1,3 @@
+module.exports = {
+  root: true,
+};

integration/templates/elements-next/.gitignore

@@ -0,0 +1,37 @@
+# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
+
+# dependencies
+/node_modules
+/.pnp
+.pnp.js
+
+# testing
+/coverage
+
+# next.js
+/.next/
+/out/
+
+# production
+/build
+
+# misc
+.DS_Store
+*.pem
+
+# debug
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+
+# local env files
+.env*.local
+
+# vercel
+.vercel
+
+# typescript
+*.tsbuildinfo
+next-env.d.ts
+
+package-lock.json

integration/templates/elements-next/README.md

@@ -0,0 +1,34 @@
+This is a [Next.js](https://nextjs.org/) project bootstrapped with [`create-next-app`](https://github.com/vercel/next.js/tree/canary/packages/create-next-app).
+
+## Getting Started
+
+First, run the development server:
+
+```bash
+npm run dev
+# or
+yarn dev
+# or
+pnpm dev
+```
+
+Open [http://localhost:3000](http://localhost:3000) with your browser to see the result.
+
+You can start editing the page by modifying `app/page.tsx`. The page auto-updates as you edit the file.
+
+This project uses [`next/font`](https://nextjs.org/docs/basic-features/font-optimization) to automatically optimize and load Inter, a custom Google Font.
+
+## Learn More
+
+To learn more about Next.js, take a look at the following resources:
+
+- [Next.js Documentation](https://nextjs.org/docs) - learn about Next.js features and API.
+- [Learn Next.js](https://nextjs.org/learn) - an interactive Next.js tutorial.
+
+You can check out [the Next.js GitHub repository](https://github.com/vercel/next.js/) - your feedback and contributions are welcome!
+
+## Deploy on Vercel
+
+The easiest way to deploy your Next.js app is to use the [Vercel Platform](https://vercel.com/new?utm_medium=default-template&filter=next.js&utm_source=create-next-app&utm_campaign=create-next-app-readme) from the creators of Next.js.
+
+Check out our [Next.js deployment documentation](https://nextjs.org/docs/deployment) for more details.

integration/templates/elements-next/next.config.js

@@ -0,0 +1,8 @@
+/** @type {import('next').NextConfig} */
+const nextConfig = {
+  eslint: {
+    ignoreDuringBuilds: true,
+  },
+};
+
+module.exports = nextConfig;