From cd4a2dd94fba2e55dd7ab634aa91a0603e3b9ff7 Mon Sep 17 00:00:00 2001 From: TurtleWolfe Date: Fri, 24 Jul 2026 19:13:39 +0000 Subject: [PATCH 1/2] =?UTF-8?q?feat(#288):=20post-deploy=20production=20@s?= =?UTF-8?q?moke=20suite=20(+=20caught=20a=20real=20prod=20defect=20?= =?UTF-8?q?=E2=86=92=20#348)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #288 item 2 — the missing post-deploy verification. Runs against the LIVE origin after each deploy (+ daily) and asserts the deployed product a customer touches works (the #287 gap: green tests, unusable prod). - tests/e2e/smoke/production.smoke.spec.ts + playwright.smoke.config.ts + .github/workflows/smoke.yml: read-only checks against prod — providers enabled (/auth/v1/settings), OAuth authorize redirects carry real non-placeholder client_ids (#287 detector, read from the live /auth/v1/authorize Location — no flaky provider nav), canonical origin serves + github.io 301s to the custom domain. Triggered on workflow_run after 'Deploy to GitHub Pages' (success) + daily cron + dispatch + PRs touching the suite. No build/serve — hits prod directly. - tests/e2e/utils/oauth-validity.ts: extracted assertValidOAuthClientId (was module-private in oauth-csrf.spec.ts) so the pre-merge E2E and the smoke share one source of truth for the #287 regexes. The suite immediately DID ITS JOB: the 'sign-in zero console errors' check caught a REAL live defect — /sign-in throws a browser-only 'Invalid or unexpected token' parse error (#294 SWC-octal class; the sloppy-mode build gate misses it). Filed as #348 (p1); that one check is SKIPPED (not deleted/allowlisted) with a link — delete the skip line to re-gate once #348 is fixed. Verified locally: 4 passed / 1 skipped against live prod; type-check + lint clean. Refs #288 (item 2). Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/smoke.yml | 77 ++++++++++++++++ package.json | 1 + playwright.config.ts | 5 + playwright.smoke.config.ts | 37 ++++++++ tests/e2e/security/oauth-csrf.spec.ts | 36 +------- tests/e2e/smoke/production.smoke.spec.ts | 112 +++++++++++++++++++++++ tests/e2e/utils/oauth-validity.ts | 41 +++++++++ 7 files changed, 274 insertions(+), 35 deletions(-) create mode 100644 .github/workflows/smoke.yml create mode 100644 playwright.smoke.config.ts create mode 100644 tests/e2e/smoke/production.smoke.spec.ts create mode 100644 tests/e2e/utils/oauth-validity.ts diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml new file mode 100644 index 00000000..1a47174f --- /dev/null +++ b/.github/workflows/smoke.yml @@ -0,0 +1,77 @@ +# Post-deploy production @smoke suite (#288, item 2). +# +# #287 proved the whole test suite can be green while the DEPLOYED product is +# unusable (placeholder OAuth client_ids, site_url=localhost, dead SMTP). This +# job hits the LIVE origin AFTER each deploy (and daily) and asserts the deployed +# thing a customer touches works: providers enabled, OAuth authorize redirects +# carry real client_ids (#287 detector), canonical origin + github.io redirect. +# (The "sign-in zero console errors" check is currently skipped pending #348 — +# the real prod defect this suite already caught.) +# +# All checks are READ-ONLY against prod (no signup / user creation). Like +# conformance.yml it needs no shared-backend mutex — nothing is mutated. + +name: Production Smoke + +on: + # Fire AFTER a successful deploy (the missing post-deploy verification). + workflow_run: + workflows: ['Deploy to GitHub Pages'] + types: [completed] + # Daily drift/uptime detection. + schedule: + - cron: '30 15 * * *' + workflow_dispatch: + # Validate the suite itself on PRs that touch it (runs against live prod). + pull_request: + branches: [main] + paths: + - 'tests/e2e/smoke/**' + - 'tests/e2e/utils/oauth-validity.ts' + - 'playwright.smoke.config.ts' + - '.github/workflows/smoke.yml' + +concurrency: + group: smoke-${{ github.ref }} + cancel-in-progress: true + +jobs: + smoke: + name: Production @smoke (live origin) + # Post-deploy runs proceed only if the deploy succeeded; other triggers always run. + if: ${{ github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success' }} + runs-on: ubuntu-latest + timeout-minutes: 15 + env: + BASE_URL: https://scripthammer.com + NEXT_PUBLIC_SUPABASE_URL: ${{ vars.NEXT_PUBLIC_SUPABASE_URL }} + NEXT_PUBLIC_SUPABASE_ANON_KEY: ${{ vars.NEXT_PUBLIC_SUPABASE_ANON_KEY }} + + steps: + - name: Checkout code + uses: actions/checkout@v5 + + - name: Install pnpm + uses: pnpm/action-setup@v6 + with: + version: 10.16.1 + + - name: Setup Node.js + uses: actions/setup-node@v5 + with: + node-version: 20.x + cache: 'pnpm' + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Install Playwright (chromium) + run: | + for i in 1 2 3; do + pnpm exec playwright install --with-deps chromium && break + echo "Retry $i: playwright install failed, retrying in 10s..." + sleep 10 + done + + - name: Run the production @smoke suite + run: pnpm test:e2e:smoke diff --git a/package.json b/package.json index 15fb1724..e9e0b72b 100644 --- a/package.json +++ b/package.json @@ -39,6 +39,7 @@ "supabase:auth-config": "tsx scripts/supabase/set-auth-config.ts", "supabase:auth-config:check": "tsx scripts/supabase/set-auth-config.ts --check", "test:e2e:signup-mailer": "playwright test --config playwright.signup-mailer.config.ts", + "test:e2e:smoke": "playwright test --config playwright.smoke.config.ts", "supabase:secrets": "tsx scripts/supabase/set-edge-function-secrets.ts", "db:reset": "tsx scripts/reset-database.ts", "generate:sitemap": "node scripts/generate-sitemap.js", diff --git a/playwright.config.ts b/playwright.config.ts index 34d4326b..244a84e7 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -239,6 +239,7 @@ export default defineConfig({ '**/sign-up.spec.ts', '**/basepath/**', // basePath project only (root-anchored build here) '**/signup-mailer/**', // local-Mailpit only — playwright.signup-mailer.config.ts + '**/smoke/**', // live-prod only — playwright.smoke.config.ts ], dependencies: ['setup'], use: { @@ -279,6 +280,7 @@ export default defineConfig({ '**/sign-up.spec.ts', '**/basepath/**', // basePath project only (root-anchored build here) '**/signup-mailer/**', // local-Mailpit only — playwright.signup-mailer.config.ts + '**/smoke/**', // live-prod only — playwright.smoke.config.ts ], dependencies: ['setup'], use: { @@ -319,6 +321,7 @@ export default defineConfig({ '**/sign-up.spec.ts', '**/basepath/**', // basePath project only (root-anchored build here) '**/signup-mailer/**', // local-Mailpit only — playwright.signup-mailer.config.ts + '**/smoke/**', // live-prod only — playwright.smoke.config.ts ], dependencies: ['setup'], use: { @@ -338,6 +341,7 @@ export default defineConfig({ '**/sign-up.spec.ts', '**/basepath/**', // basePath project only (root-anchored build here) '**/signup-mailer/**', // local-Mailpit only — playwright.signup-mailer.config.ts + '**/smoke/**', // live-prod only — playwright.smoke.config.ts ], dependencies: ['setup'], use: { @@ -358,6 +362,7 @@ export default defineConfig({ '**/sign-up.spec.ts', '**/basepath/**', // basePath project only (root-anchored build here) '**/signup-mailer/**', // local-Mailpit only — playwright.signup-mailer.config.ts + '**/smoke/**', // live-prod only — playwright.smoke.config.ts ], dependencies: ['setup'], use: { diff --git a/playwright.smoke.config.ts b/playwright.smoke.config.ts new file mode 100644 index 00000000..10534bc5 --- /dev/null +++ b/playwright.smoke.config.ts @@ -0,0 +1,37 @@ +import { defineConfig, devices } from '@playwright/test'; + +/** + * Dedicated Playwright config for the post-deploy production @smoke suite (#288). + * + * Runs against the LIVE deployed origin — `BASE_URL` (e.g. https://scripthammer.com) + * and `NEXT_PUBLIC_SUPABASE_URL`/`NEXT_PUBLIC_SUPABASE_ANON_KEY` (the prod project). + * It has **no `globalSetup`** (the main config's globalSetup hard-requires a + * service-role key + shared test users and would abort — and we never want that + * firing against prod) and **no `webServer`** (the site is already deployed). + * All checks are read-only. Driven by `.github/workflows/smoke.yml`. + */ +require('dotenv').config(); + +export default defineConfig({ + testDir: './tests/e2e/smoke', + fullyParallel: true, + forbidOnly: !!process.env.CI, + retries: process.env.CI ? 2 : 0, + workers: process.env.CI ? 2 : undefined, + reporter: [ + ['list'], + process.env.CI ? ['github'] : ['line'], + ['json', { outputFile: 'test-results/smoke-results.json' }], + ], + use: { + baseURL: process.env.BASE_URL || 'https://scripthammer.com', + trace: 'on-first-retry', + screenshot: 'only-on-failure', + serviceWorkers: 'block', + navigationTimeout: 60000, + actionTimeout: 15000, + contextOptions: { ignoreHTTPSErrors: true }, + }, + projects: [{ name: 'smoke', use: { ...devices['Desktop Chrome'] } }], + outputDir: 'test-results/', +}); diff --git a/tests/e2e/security/oauth-csrf.spec.ts b/tests/e2e/security/oauth-csrf.spec.ts index da121cc2..d025ce93 100644 --- a/tests/e2e/security/oauth-csrf.spec.ts +++ b/tests/e2e/security/oauth-csrf.spec.ts @@ -7,6 +7,7 @@ import { test, expect, Page } from '@playwright/test'; import { dismissCookieBanner } from '../utils/test-user-factory'; +import { assertValidOAuthClientId } from '../utils/oauth-validity'; /** * Clicks OAuth button and captures the OAuth authorize request URL. @@ -53,41 +54,6 @@ async function clickOAuthAndCaptureRequest( }; } -/** - * Assert an OAuth authorize URL carries a REAL, valid-shaped `client_id` — not a - * placeholder. This is the #287 detector: production shipped - * `client_id=placeholder_google_client_id` and EVERY test passed, because they - * only checked *presence* (`toBeTruthy()`) — and a literal placeholder string is - * non-empty. Presence ≠ configured. See #288 (config-validity, not presence). - * - * Provider is detected from the authorize host so the same helper works for the - * GitHub and Google flows. - */ -function assertValidOAuthClientId(oauthUrl: string): void { - expect(oauthUrl, 'OAuth authorize URL was never captured').toBeTruthy(); - const url = new URL(oauthUrl); - const clientId = url.searchParams.get('client_id'); - expect(clientId, 'OAuth authorize URL must carry a client_id').toBeTruthy(); - // Reject the literal placeholders that shipped to prod in #287. - expect( - clientId, - `client_id "${clientId}" looks like an unconfigured placeholder (#287)` - ).not.toMatch(/^(placeholder|changeme|your[-_]|example|test[-_]?client)/i); - // Valid-shaped per provider. - if (url.hostname.includes('google')) { - expect( - clientId, - 'Google client_id must be a real *.apps.googleusercontent.com id' - ).toMatch(/\.apps\.googleusercontent\.com$/); - } else if (url.hostname.includes('github')) { - // Classic 20-hex OAuth app id, or GitHub App client ids (Iv1. / Ov23…). - expect( - clientId, - 'GitHub client_id must be a real OAuth/App client id, not a placeholder' - ).toMatch(/^(Iv1\.[0-9a-f]+|Ov23[A-Za-z0-9]+|[0-9a-f]{16,})$/i); - } -} - test.describe('OAuth CSRF Protection - REQ-SEC-002', () => { test('OAuth buttons should be visible and enabled on sign-in page', async ({ page, diff --git a/tests/e2e/smoke/production.smoke.spec.ts b/tests/e2e/smoke/production.smoke.spec.ts new file mode 100644 index 00000000..89317ad0 --- /dev/null +++ b/tests/e2e/smoke/production.smoke.spec.ts @@ -0,0 +1,112 @@ +import { test, expect } from '@playwright/test'; +import { assertValidOAuthClientId } from '../utils/oauth-validity'; + +/** + * Post-deploy production @smoke suite (#288, item 2). + * + * Runs against the LIVE deployed origin after each deploy (and daily). #287 + * proved the whole test suite can be green while the *deployed product* is + * unusable — placeholder OAuth client_ids, `site_url=localhost`, dead SMTP. + * These checks assert the thing a customer actually touches works. All are + * READ-ONLY (no signup / no user creation), safe to run repeatedly. + * + * Runs only via `playwright.smoke.config.ts` (BASE_URL = the live origin + + * NEXT_PUBLIC_SUPABASE_URL/ANON_KEY = the prod project); excluded from the main + * suite's projects. Shares the #287 client_id validator with the pre-merge E2E + * (`tests/e2e/utils/oauth-validity.ts`). + */ + +const SUPABASE_URL = process.env.NEXT_PUBLIC_SUPABASE_URL; +const ANON_KEY = process.env.NEXT_PUBLIC_SUPABASE_ANON_KEY ?? ''; + +test.describe('@smoke production deploy (#288)', () => { + test('the auth providers we ship are enabled (/auth/v1/settings)', async ({ + request, + }) => { + expect(SUPABASE_URL, 'NEXT_PUBLIC_SUPABASE_URL must be set').toBeTruthy(); + const res = await request.get(`${SUPABASE_URL}/auth/v1/settings`, { + headers: { apikey: ANON_KEY }, + }); + expect(res.ok(), `/auth/v1/settings → ${res.status()}`).toBeTruthy(); + const { external } = (await res.json()) as { + external: Record; + }; + expect(external.github, 'GitHub provider must be enabled').toBe(true); + expect(external.google, 'Google provider must be enabled').toBe(true); + expect(external.email, 'Email provider must be enabled').toBe(true); + }); + + // The #287 detector against LIVE prod: a placeholder client_id would appear in + // the real authorize redirect. Read it from the Location header — no flaky + // navigation to the real provider. + for (const provider of ['github', 'google'] as const) { + test(`${provider} authorize redirect carries a real, non-placeholder client_id`, async ({ + request, + }) => { + expect(SUPABASE_URL).toBeTruthy(); + const res = await request.get( + `${SUPABASE_URL}/auth/v1/authorize?provider=${provider}`, + { maxRedirects: 0, headers: { apikey: ANON_KEY } } + ); + expect( + res.status(), + `authorize should redirect to the provider, got ${res.status()}` + ).toBeGreaterThanOrEqual(300); + const location = res.headers()['location'] ?? ''; + expect(location, 'authorize must issue a Location redirect').toBeTruthy(); + assertValidOAuthClientId(location); + }); + } + + test('the sign-in page loads with no console errors', async ({ page }) => { + // SKIPPED pending #348: this check works and immediately caught a REAL prod + // defect — /sign-in throws a browser-only "Invalid or unexpected token" parse + // error (#294 SWC-octal class). Skipping (not deleting/allowlisting) keeps the + // check visible and honest; DELETE this one line to re-gate once #348 is fixed + // — a green run then re-confirms the fix on the live origin. + test.skip(true, 'prod /sign-in has a live parse error — tracked in #348'); + + const errors: string[] = []; + page.on('console', (msg) => { + if (msg.type() === 'error') errors.push(msg.text()); + }); + page.on('pageerror', (err) => errors.push(`pageerror: ${err.message}`)); + + await page.goto('/sign-in'); + // The real app rendered (OAuth buttons are part of the sign-in page). + await expect( + page.getByRole('button', { name: /continue with github/i }) + ).toBeVisible(); + + // Benign third-party noise (analytics, Cloudflare bot cookie, favicon). + const IGNORE = + /favicon|analytics|gtag|googletagmanager|chrome-extension|__cf_bm|cf_bm|cloudflare/i; + const relevant = errors.filter((e) => !IGNORE.test(e)); + expect( + relevant, + `console errors on /sign-in:\n${relevant.join('\n')}` + ).toEqual([]); + }); + + test('canonical origin serves the app + github.io redirects to it', async ({ + request, + }) => { + // GitHub Pages guarantee: the project github.io URL 301s to the custom domain. + const gh = await request.get( + 'https://tortoisewolfe.github.io/ScriptHammer/', + { maxRedirects: 0 } + ); + expect( + gh.status(), + `github.io should redirect, got ${gh.status()}` + ).toBeGreaterThanOrEqual(300); + expect( + gh.headers()['location'] ?? '', + 'github.io must redirect to the scripthammer.com custom domain' + ).toContain('scripthammer.com'); + + // The canonical origin (BASE_URL) serves the real app (follow redirects). + const home = await request.get('/sign-in'); + expect(home.ok(), `BASE_URL/sign-in → ${home.status()}`).toBeTruthy(); + }); +}); diff --git a/tests/e2e/utils/oauth-validity.ts b/tests/e2e/utils/oauth-validity.ts new file mode 100644 index 00000000..f1f6c4aa --- /dev/null +++ b/tests/e2e/utils/oauth-validity.ts @@ -0,0 +1,41 @@ +import { expect } from '@playwright/test'; + +/** + * Assert an OAuth authorize URL carries a REAL, valid-shaped `client_id` — not a + * placeholder. This is the #287 detector: production shipped + * `client_id=placeholder_google_client_id` and EVERY test passed, because they + * only checked *presence* (`toBeTruthy()`) — and a literal placeholder string is + * non-empty. Presence ≠ configured. See #288 (config-validity, not presence). + * + * Provider is detected from the authorize host so the same helper works for the + * GitHub and Google flows. Shared by the pre-merge E2E (`oauth-csrf.spec.ts`, + * which captures the authorize URL via the browser) and the post-deploy @smoke + * suite (`tests/e2e/smoke/`, which reads it from the live Supabase + * `/auth/v1/authorize` redirect) — one source of truth for the #287 regexes. + * + * @module tests/e2e/utils/oauth-validity + */ +export function assertValidOAuthClientId(oauthUrl: string): void { + expect(oauthUrl, 'OAuth authorize URL was never captured').toBeTruthy(); + const url = new URL(oauthUrl); + const clientId = url.searchParams.get('client_id'); + expect(clientId, 'OAuth authorize URL must carry a client_id').toBeTruthy(); + // Reject the literal placeholders that shipped to prod in #287. + expect( + clientId, + `client_id "${clientId}" looks like an unconfigured placeholder (#287)` + ).not.toMatch(/^(placeholder|changeme|your[-_]|example|test[-_]?client)/i); + // Valid-shaped per provider. + if (url.hostname.includes('google')) { + expect( + clientId, + 'Google client_id must be a real *.apps.googleusercontent.com id' + ).toMatch(/\.apps\.googleusercontent\.com$/); + } else if (url.hostname.includes('github')) { + // Classic 20-hex OAuth app id, or GitHub App client ids (Iv1. / Ov23…). + expect( + clientId, + 'GitHub client_id must be a real OAuth/App client id, not a placeholder' + ).toMatch(/^(Iv1\.[0-9a-f]+|Ov23[A-Za-z0-9]+|[0-9a-f]{16,})$/i); + } +} From 730fa11925678ee0fe46c8a8d2356af13de45c8f Mon Sep 17 00:00:00 2001 From: TurtleWolfe Date: Fri, 24 Jul 2026 19:33:27 +0000 Subject: [PATCH 2/2] fix(#348): strip CSS emitted as