improve test handling, prepares for e2e runs (#514)

apps/discord-bot-frontend/.gitignore

@@ -1,2 +1,7 @@
-.svelte-kit
-build
+# ignore SvelteKit output
+/.svelte-kit
+/build
+# ignore test state files
+/playwright/*
+# ignore test report files
+/playwright-report

apps/discord-bot-frontend/package.json

@@ -12,7 +12,7 @@
     "test": "playwright test",
     "setup-dev": "prisma db push",
     "setup-test": "DATABASE_URL=\"file:data/test.db\" prisma db push",
-    "postinstall": "prisma generate; playwright install",
+    "postinstall": "prisma generate; playwright install --with-deps",
     "preview": "vite preview --port 3000",
     "seed": "vite-node ./scripts/seed.ts",
     "prepare": "svelte-kit sync"

apps/discord-bot-frontend/playwright.config.ts

@@ -1,19 +1,39 @@
+import { defineConfig, devices } from '@playwright/test'
 import { loadEnvVars } from './vite.config'
-import type { PlaywrightTestConfig } from '@playwright/test'
+import { z } from 'zod'
 
 loadEnvVars()
-process.env.IS_TEST = 'true'
 
-const config: PlaywrightTestConfig = {
+// define the base set of env vars needed to run tests
+// tests that are opt-in (like webhook tests) should not have their env vars set here
+const env_schema = z.object({
+  TEST_HOST: z.string().default('http://localhost:3000/'),
+  DISCORD_GUILD_ID: z.string(),
+  // add more required env vars here as necessary
+})
+// ensure necessary env vars are available
+export const env = env_schema.parse(process.env)
+
+export default defineConfig({
+  forbidOnly: !!process.env.CI,
+  reporter: process.env.CI
+    ? [['html', { open: 'never' }]]
+    : [['html', { open: 'on-failure' }]],
+  workers: process.env.CI ? 1 : undefined,
   webServer: {
-    command: 'pnpm run build-app && pnpm run preview',
+    reuseExistingServer: !process.env.CI,
+    command: 'pnpm run preview',
     port: 3000,
   },
+  projects: [
+    {
+      use: devices['Desktop Chrome'],
+      retries: 0,
+    },
+  ],
   testDir: 'tests',
-  testMatch: /(.+\.)?(test|spec)\.[jt]s/,
+  testMatch: '**/*.test.ts',
   use: {
-    baseURL: 'http://localhost:3000/',
+    baseURL: env.TEST_HOST,
   },
-}
-
-export default config
+})

apps/discord-bot-frontend/tests/api/webhooks/github-org-membership.test.ts

@@ -8,6 +8,16 @@ import {
   removedPayloadUserDNE,
 } from '../../mock/github-webhook'
 
+test.skip(
+  ({ baseURL }) => !baseURL?.startsWith('http://localhost'),
+  'Skip in live environments'
+)
+
+test.fail(
+  !process.env.GITHUB_WEBHOOK_SECRET,
+  'GITHUB_WEBHOOK_SECRET is not set'
+)
+
 test.describe('POST api/webhooks/github-org-membership', () => {
   test.describe('webhook verification', () => {
     test('should return true with payload for added member', () => {
@@ -90,22 +100,25 @@ test.describe('POST api/webhooks/github-org-membership', () => {
     expect(response.status()).toBe(403)
   })
 
-  /**
-   * @TODO fix this in CI, test runs fine locally
-   */
-  // test('should return 201 if everything is correct', async () => {
-  //   const response = await request(app)
-  //     .post('/api/webhooks/github-org-membership')
-  //     .send(addedPayload1.body)
-  //     .set(addedPayload1.headers)
-  //   expect(response.status).toBe(201)
-  // })
+  test.fixme(
+    'should return 201 if everything is correct',
+    async ({ request }) => {
+      const response = await request.post(
+        '/api/webhooks/github-org-membership',
+        {
+          data: addedPayload1.body,
+          headers: addedPayload1.headers,
+        }
+      )
+      expect(response.status).toBe(201)
+    }
+  )
 
-  // test(`should return 403 if user isn't in db`, async ({ request }) => {
-  //   const response = await request.post('/api/webhooks/github-org-membership', {
-  //     data: addedPayloadUserDNE.body,
-  //     headers: addedPayloadUserDNE.headers,
-  //   })
-  //   expect(response.status()).toBe(403)
-  // })
+  test.fixme(`should return 403 if user isn't in db`, async ({ request }) => {
+    const response = await request.post('/api/webhooks/github-org-membership', {
+      data: addedPayloadUserDNE.body,
+      headers: addedPayloadUserDNE.headers,
+    })
+    expect(response.status()).toBe(403)
+  })
 })

apps/discord-bot-frontend/tests/api/webhooks/github-release.test.ts

@@ -1,6 +1,16 @@
 import { test, expect } from '@playwright/test'
 import { mockedCreated, mockedReleased } from '../../mock/github-webhook'
 
+test.skip(
+  ({ baseURL }) => !baseURL?.startsWith('http://localhost'),
+  'Skip in live environments'
+)
+
+test.fail(
+  !process.env.DISCORD_WEBHOOK_URL_RELEASES,
+  'DISCORD_WEBHOOK_URL_RELEASES is not set'
+)
+
 test.describe('GitHub Release webhook', () => {
   test('should not return 401', async ({ request }) => {
     const response = await request.post('/api/webhooks/github-release', {
@@ -24,8 +34,7 @@ test.describe('GitHub Release webhook', () => {
     expect(response.status()).toBe(403)
   })
 
-  /** @todo unskip */
-  test.skip('should return 400 if webhook URL is bad', async ({ request }) => {
+  test.fixme('should return 400 if webhook URL is bad', async ({ request }) => {
     const url = process.env.DISCORD_WEBHOOK_URL_RELEASES
     process.env.DISCORD_WEBHOOK_URL_RELEASES =
       'https://discordapp.com/api/webhooks/bad'
@@ -49,17 +58,17 @@ test.describe('GitHub Release webhook', () => {
     expect(response.status()).toBe(400)
   })
 
-  /** @todo unskip */
-  test.skip('should return 201 if everything is correct', async ({
-    request,
-  }) => {
-    const response = await request.post('/api/webhooks/github-release', {
-      headers: mockedReleased.headers,
-      data: mockedReleased.body,
-    })
+  test.fixme(
+    'should return 201 if everything is correct',
+    async ({ request }) => {
+      const response = await request.post('/api/webhooks/github-release', {
+        headers: mockedReleased.headers,
+        data: mockedReleased.body,
+      })
 
-    expect(response.status()).toBe(201)
-  })
+      expect(response.status()).toBe(201)
+    }
+  )
 
   test(`should return 204 is event action is not 'released'`, async ({
     request,

apps/discord-bot-frontend/tests/guild-cookie.test.ts

@@ -1,5 +1,6 @@
 import { test, expect } from '@playwright/test'
 import { GUILD_COOKIE } from '$lib/constants'
+import { env } from '../playwright.config'
 
 test.describe('guild cookie', () => {
   test('sets on page load', async ({ page }) => {
@@ -12,7 +13,7 @@ test.describe('guild cookie', () => {
     await page.goto('/')
     const cookies = await page.context().cookies()
     const cookie = cookies.find((c) => c.name === GUILD_COOKIE)
-    expect(cookie?.value).toEqual(process.env.DISCORD_GUILD_ID)
+    expect(cookie?.value).toEqual(env.DISCORD_GUILD_ID)
   })
 
   test('falls back to default value if invalid', async ({ page }) => {
@@ -31,6 +32,6 @@ test.describe('guild cookie', () => {
     // check the cookies after loading the page
     const cookies = await page.context().cookies()
     const cookie = cookies.find((c) => c.name === GUILD_COOKIE)
-    expect(cookie?.value).toEqual(process.env.DISCORD_GUILD_ID)
+    expect(cookie?.value).toEqual(env.DISCORD_GUILD_ID)
   })
 })

apps/discord-bot-frontend/tests/setup/seed.ts

@@ -1,6 +1,7 @@
 import { PrismaClient } from '@prisma/client'
 import { ACCESS_LEVELS } from '$lib/constants'
 import { init } from '$lib/db'
+import { env } from '../../playwright.config'
 const prisma = new PrismaClient()
 
 export async function seed() {
@@ -36,7 +37,7 @@ export async function seed() {
   const STAFF_ROLE = '1001228846768590934'
   const CONTRIBUTOR_ROLE = '1001228846768590931'
   await prisma.configuration.upsert({
-    where: { id: import.meta.env.VITE_DISCORD_GUILD_ID },
+    where: { id: env.DISCORD_GUILD_ID },
     update: {},
     create: {
       name: 'hey-amplify-e2e',
@@ -75,8 +76,8 @@ export async function seed() {
       },
       guild: {
         connectOrCreate: {
-          where: { id: import.meta.env.VITE_DISCORD_GUILD_ID },
-          create: { id: import.meta.env.VITE_DISCORD_GUILD_ID },
+          where: { id: env.DISCORD_GUILD_ID },
+          create: { id: env.DISCORD_GUILD_ID },
         },
       },
     },