test(e2e): Do not require DSN to run E2E tests (#12743)

First, this means it is easier to run this locally. 

Secondly, we do not even need to send data to Sentry anymore, so we can
adjust the proxy server to stop doing this.

Lastly, this means that we can also safely run E2E tests for external
contributors.

Closes #11910

Author: mydea

.github/workflows/build.yml

@@ -971,22 +971,19 @@ jobs:
 
   job_e2e_tests:
     name: E2E ${{ matrix.label || matrix.test-application }} Test
-    # We only run E2E tests for non-fork PRs because the E2E tests require secrets to work and they can't be accessed from forks
     # We need to add the `always()` check here because the previous step has this as well :(
     # See: https://github.com/actions/runner/issues/2205
-    if:
-      always() && needs.job_e2e_prepare.result == 'success' &&
-      (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
+    if: always() && needs.job_e2e_prepare.result == 'success'
     needs: [job_get_metadata, job_build, job_e2e_prepare]
     runs-on: ubuntu-20.04
     timeout-minutes: 10
     env:
-      E2E_TEST_AUTH_TOKEN: ${{ secrets.E2E_TEST_AUTH_TOKEN }}
-      E2E_TEST_DSN: ${{ secrets.E2E_TEST_DSN }}
+      # We just use a dummy DSN here, only send to the tunnel anyhow
+      E2E_TEST_DSN: 'https://username@domain/123'
       # Needed because some apps expect a certain prefix
-      NEXT_PUBLIC_E2E_TEST_DSN: ${{ secrets.E2E_TEST_DSN }}
-      PUBLIC_E2E_TEST_DSN: ${{ secrets.E2E_TEST_DSN }}
-      REACT_APP_E2E_TEST_DSN: ${{ secrets.E2E_TEST_DSN }}
+      NEXT_PUBLIC_E2E_TEST_DSN: 'https://username@domain/123'
+      PUBLIC_E2E_TEST_DSN: 'https://username@domain/123'
+      REACT_APP_E2E_TEST_DSN: 'https://username@domain/123'
       E2E_TEST_SENTRY_ORG_SLUG: 'sentry-javascript-sdks'
       E2E_TEST_SENTRY_PROJECT: 'sentry-javascript-e2e-tests'
     strategy:
@@ -1072,7 +1069,7 @@ jobs:
           ref: ${{ env.HEAD_COMMIT }}
       - uses: pnpm/action-setup@v4
         with:
-          version: 8.3.1
+          version: 9.4.0
       - name: Set up Node
         uses: actions/setup-node@v4
         with:
@@ -1109,12 +1106,12 @@ jobs:
       - name: Build E2E app
         working-directory: dev-packages/e2e-tests/test-applications/${{ matrix.test-application }}
         timeout-minutes: 5
-        run: yarn ${{ matrix.build-command || 'test:build' }}
+        run: pnpm ${{ matrix.build-command || 'test:build' }}
 
       - name: Run E2E test
         working-directory: dev-packages/e2e-tests/test-applications/${{ matrix.test-application }}
         timeout-minutes: 5
-        run: yarn test:assert
+        run: pnpm test:assert
 
       - name: Deploy Astro to Cloudflare
         uses: cloudflare/pages-action@v1
@@ -1149,8 +1146,6 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        is_dependabot:
-          - ${{ github.actor == 'dependabot[bot]' }}
         test-application:
           [
             'react-send-to-sentry',
@@ -1159,8 +1154,17 @@ jobs:
           ]
         build-command:
           - false
+        assert-command:
+          - false
         label:
           - false
+        include:
+          - test-application: 'create-remix-app'
+            assert-command: 'test:assert-sourcemaps'
+            label: 'create-remix-app (sourcemaps)'
+          - test-application: 'create-remix-app-legacy'
+            assert-command: 'test:assert-sourcemaps'
+            label: 'create-remix-app-legacy (sourcemaps)'
 
     steps:
       - name: Check out current commit (${{ needs.job_get_metadata.outputs.commit_label }})
@@ -1169,7 +1173,7 @@ jobs:
           ref: ${{ env.HEAD_COMMIT }}
       - uses: pnpm/action-setup@v4
         with:
-          version: 8.3.1
+          version: 9.4.0
       - name: Set up Node
         uses: actions/setup-node@v4
         with:
@@ -1203,12 +1207,12 @@ jobs:
       - name: Build E2E app
         working-directory: dev-packages/e2e-tests/test-applications/${{ matrix.test-application }}
         timeout-minutes: 5
-        run: yarn ${{ matrix.build-command || 'test:build' }}
+        run: pnpm ${{ matrix.build-command || 'test:build' }}
 
       - name: Run E2E test
         working-directory: dev-packages/e2e-tests/test-applications/${{ matrix.test-application }}
         timeout-minutes: 5
-        run: yarn test:assert
+        run: pnpm ${{ matrix.assert-command || 'test:assert' }}
 
   job_profiling_e2e_tests:
     name: E2E ${{ matrix.label || matrix.test-application }} Test

.github/workflows/canary.yml

@@ -56,12 +56,12 @@ jobs:
     runs-on: ubuntu-20.04
     timeout-minutes: 20
     env:
-      E2E_TEST_AUTH_TOKEN: ${{ secrets.E2E_TEST_AUTH_TOKEN }}
-      E2E_TEST_DSN: ${{ secrets.E2E_TEST_DSN }}
-      # Needed because certain apps expect a certain prefix
-      NEXT_PUBLIC_E2E_TEST_DSN: ${{ secrets.E2E_TEST_DSN }}
-      PUBLIC_E2E_TEST_DSN: ${{ secrets.E2E_TEST_DSN }}
-      REACT_APP_E2E_TEST_DSN: ${{ secrets.E2E_TEST_DSN }}
+      # We just use a dummy DSN here, only send to the tunnel anyhow
+      E2E_TEST_DSN: 'https://username@domain/123'
+      # Needed because some apps expect a certain prefix
+      NEXT_PUBLIC_E2E_TEST_DSN: 'https://username@domain/123'
+      PUBLIC_E2E_TEST_DSN: 'https://username@domain/123'
+      REACT_APP_E2E_TEST_DSN: 'https://username@domain/123'
       E2E_TEST_SENTRY_ORG_SLUG: 'sentry-javascript-sdks'
       E2E_TEST_SENTRY_PROJECT: 'sentry-javascript-e2e-tests'
     strategy:

dev-packages/e2e-tests/README.md

@@ -8,7 +8,8 @@ current state.
 Prerequisites: Docker
 
 - Copy `.env.example` to `.env`
-- Fill in auth information in `.env` for an example Sentry project
+- OPTIONAL: Fill in auth information in `.env` for an example Sentry project - you only need this to run E2E tests that
+  send data to Sentry.
 - Run `yarn build:tarball` in the root of the repository
 
 To finally run all of the tests:

dev-packages/e2e-tests/lib/validate.ts

@@ -1,17 +1,12 @@
 /* eslint-disable no-console */
 import * as dotenv from 'dotenv';
 
-import { validate } from './lib/validate';
 import { registrySetup } from './registrySetup';
 
 async function run(): Promise<void> {
   // Load environment variables from .env file locally
   dotenv.config();
 
-  if (!validate()) {
-    process.exit(1);
-  }
-
   try {
     registrySetup();
   } catch (error) {

dev-packages/e2e-tests/prepare.ts

@@ -4,9 +4,12 @@ import { resolve } from 'path';
 import * as dotenv from 'dotenv';
 import { sync as globSync } from 'glob';
 
-import { validate } from './lib/validate';
 import { registrySetup } from './registrySetup';
 
+const DEFAULT_DSN = 'https://username@domain/123';
+const DEFAULT_SENTRY_ORG_SLUG = 'sentry-javascript-sdks';
+const DEFAULT_SENTRY_PROJECT = 'sentry-javascript-e2e-tests';
+
 function asyncExec(command: string, options: { env: Record<string, string | undefined>; cwd: string }): Promise<void> {
   return new Promise((resolve, reject) => {
     const process = spawn(command, { ...options, shell: true });
@@ -39,17 +42,21 @@ async function run(): Promise<void> {
   // Allow to run a single app only via `yarn test:run <app-name>`
   const appName = process.argv[2];
 
-  if (!validate()) {
-    process.exit(1);
-  }
+  const dsn = process.env.E2E_TEST_DSN || DEFAULT_DSN;
 
   const envVarsToInject = {
-    NEXT_PUBLIC_E2E_TEST_DSN: process.env.E2E_TEST_DSN,
-    PUBLIC_E2E_TEST_DSN: process.env.E2E_TEST_DSN,
-    REACT_APP_E2E_TEST_DSN: process.env.E2E_TEST_DSN,
+    E2E_TEST_DSN: dsn,
+    NEXT_PUBLIC_E2E_TEST_DSN: dsn,
+    PUBLIC_E2E_TEST_DSN: dsn,
+    REACT_APP_E2E_TEST_DSN: dsn,
+    E2E_TEST_SENTRY_ORG_SLUG: process.env.E2E_TEST_SENTRY_ORG_SLUG || DEFAULT_SENTRY_ORG_SLUG,
+    E2E_TEST_SENTRY_PROJECT: process.env.E2E_TEST_SENTRY_PROJECT || DEFAULT_SENTRY_PROJECT,
   };
 
-  const env = { ...process.env, ...envVarsToInject };
+  const env = {
+    ...process.env,
+    ...envVarsToInject,
+  };
 
   try {
     console.log('Cleaning test-applications...');

dev-packages/e2e-tests/run.ts

@@ -3,5 +3,4 @@ import { startEventProxyServer } from '@sentry-internal/test-utils';
 startEventProxyServer({
   port: 3031,
   proxyServerName: 'aws-serverless-lambda-layer-cjs',
-  forwardToSentry: false,
 });

dev-packages/e2e-tests/test-applications/aws-lambda-layer-cjs/start-event-proxy.mjs

@@ -2,13 +2,15 @@
   "private": true,
   "sideEffects": false,
   "scripts": {
-    "build": "remix build --sourcemap && ./upload-sourcemaps.sh",
+    "build": "remix build --sourcemap",
+    "upload-sourcemaps": "./upload-sourcemaps.sh",
     "dev": "remix dev",
     "start": "remix-serve build",
     "typecheck": "tsc",
     "clean": "npx rimraf node_modules pnpm-lock.yaml",
     "test:build": "pnpm install && npx playwright install && pnpm build",
-    "test:assert": "pnpm playwright test"
+    "test:assert": "pnpm playwright test",
+    "test:assert-sourcemaps": "pnpm upload-sourcemaps"
   },
   "dependencies": {
     "@sentry/remix": "latest || *",

dev-packages/e2e-tests/test-applications/create-remix-app-legacy/package.json

@@ -2,13 +2,15 @@
   "private": true,
   "sideEffects": false,
   "scripts": {
-    "build": "remix build --sourcemap && ./upload-sourcemaps.sh",
+    "build": "remix build --sourcemap",
+    "upload-sourcemaps": "./upload-sourcemaps.sh",
     "dev": "remix dev",
     "start": "NODE_OPTIONS='--require=./instrument.server.cjs' remix-serve build",
     "typecheck": "tsc",
     "clean": "npx rimraf node_modules pnpm-lock.yaml",
     "test:build": "pnpm install && npx playwright install && pnpm build",
-    "test:assert": "pnpm playwright test"
+    "test:assert": "pnpm playwright test",
+    "test:assert-sourcemaps": "pnpm upload-sourcemaps"
   },
   "dependencies": {
     "@sentry/remix": "latest || *",

dev-packages/e2e-tests/test-applications/create-remix-app/package.json

@@ -6,9 +6,7 @@ import Index from './pages/Index';
 
 Sentry.init({
   environment: 'qa', // dynamic sampling bias to keep transactions
-  dsn:
-    process.env.REACT_APP_E2E_TEST_DSN ||
-    'https://3b6c388182fb435097f41d181be2b2ba@o4504321058471936.ingest.sentry.io/4504321066008576',
+  dsn: process.env.REACT_APP_E2E_TEST_DSN,
   release: 'e2e-test',
   tunnel: 'http://localhost:3031/', // proxy server
 });