feat: Initial E2E test framework for v2 (#288)

This brings an initial E2E test (really, an integration test - it's only running the server locally, as opposed to against a deployment - but it'd be easy to point playwright to a deployment).

Demo gif:
![test2](https://user-images.githubusercontent.com/88213859/156078517-6cb4ef84-337b-4e16-a8bc-aea7d06dcbcb.gif)

This test exercises a minimal flow for login:
- Run the `coderd` binary to start a server on 3000
- Create an initial user as part of setup
- Go through the login flow and verify we land on the projects page

It will be useful to have to ensure that #360 doesn't introduce a regression in the login flow

Future E2E tests that would be useful:
- Create a project & verify it shows in the UI
- Create a workspace and verify it shows in the UI

.github/workflows/coder.yaml

@@ -164,6 +164,7 @@ jobs:
         env:
           DATADOG_API_KEY: ${{ secrets.DATADOG_API_KEY }}
           DD_DATABASE: fake
+          DD_CATEGORY: unit
           GIT_COMMIT_MESSAGE: ${{ github.event.head_commit.message }}
         run: go run scripts/datadog-cireport/main.go gotests.xml
 
@@ -279,6 +280,74 @@ jobs:
         if: (success() || failure()) && github.actor != 'dependabot[bot]'
         env:
           DATADOG_API_KEY: ${{ secrets.DATADOG_API_KEY }}
-          DD_DATABASE: postgresql
+          DD_CATEGORY: unit
+          GIT_COMMIT_MESSAGE: ${{ github.event.head_commit.message }}
+        run: go run scripts/datadog-cireport/main.go site/test-results/junit.xml
+
+  test-e2e:
+    name: "test/e2e/${{ matrix.os }}"
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os:
+          - ubuntu-latest
+          - macos-latest
+          # TODO: Get `make build` running on Windows 2022
+          # https://github.com/coder/coder/issues/384
+          # - windows-2022
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Cache Node
+        id: cache-node
+        uses: actions/cache@v2
+        with:
+          path: |
+            **/node_modules
+            .eslintcache
+          key: js-${{ runner.os }}-test-${{ hashFiles('**/yarn.lock') }}
+
+      # Go is required for uploading the test results to datadog
+      - uses: actions/setup-go@v2
+        with:
+          go-version: "^1.17"
+
+      - uses: hashicorp/setup-terraform@v1
+        with:
+          terraform_version: 1.1.2
+          terraform_wrapper: false
+
+      - uses: actions/setup-node@v3
+        with:
+          node-version: "14"
+
+      - uses: actions/cache@v2
+        with:
+          # Go mod cache, Linux build cache, Mac build cache, Windows build cache
+          path: |
+            ~/go/pkg/mod
+            ~/.cache/go-build
+            ~/Library/Caches/go-build
+            %LocalAppData%\go-build
+          key: ${{ matrix.os }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ matrix.os }}-go-
+
+      - run: make build
+
+      - run: yarn playwright:install
+        working-directory: site
+
+      - run: yarn playwright:install-deps
+        working-directory: site
+
+      - run: yarn playwright:test
+        working-directory: site
+
+      - name: Upload DataDog Trace
+        if: (success() || failure()) && github.actor != 'dependabot[bot]' && runner.os == 'Linux'
+        env:
+          DATADOG_API_KEY: ${{ secrets.DATADOG_API_KEY }}
+          DD_CATEGORY: e2e
           GIT_COMMIT_MESSAGE: ${{ github.event.head_commit.message }}
-        run: go run scripts/datadog-cireport/main.go site/test_results/junit.xml
+        run: go run scripts/datadog-cireport/main.go site/test-results/junit.xml

.gitignore

@@ -20,7 +20,7 @@ site/.eslintcache
 site/.next/
 site/node_modules/
 site/storybook-static/
-site/test_results/
+site/test-results/
 site/yarn-error.log
 coverage/
 

scripts/datadog-cireport/main.go

@@ -75,7 +75,8 @@ func main() {
 		"service":              "coder",
 		"_dd.cireport_version": "2",
 
-		"test.traits": fmt.Sprintf(`{"database":["%s"]}`, os.Getenv("DD_DATABASE")),
+		"test.traits": fmt.Sprintf(`{"database":["%s"], "category":["%s"]}`,
+			os.Getenv("DD_DATABASE"), os.Getenv("DD_CATEGORY")),
 
 		// Additional tags found in DataDog docs. See:
 		// https://docs.datadoghq.com/continuous_integration/setup_tests/junit_upload/#collecting-environment-configuration-metadata

site/.eslintignore

@@ -7,4 +7,4 @@ out
 coverage
 .next
 storybook-static
-test_results
+test-results

site/.prettierignore

@@ -15,4 +15,4 @@ yarn-error.log
 coverage/
 out/
 storybook-static/
-test_results/
+test-results/

site/e2e/constants.ts

@@ -0,0 +1,5 @@
+// Default credentials and user for running tests
+export const username = "admin"
+export const password = "password"
+export const organization = "acme-crop"
+export const email = "admin@coder.com"

site/e2e/globalSetup.ts

@@ -0,0 +1,24 @@
+import { FullConfig, request } from "@playwright/test"
+import { email, username, password, organization } from "./constants"
+
+const globalSetup = async (config: FullConfig): Promise<void> => {
+  // Grab the 'baseURL' from the webserver (`coderd`)
+  const { baseURL } = config.projects[0].use
+
+  // Create a context that will issue http requests.
+  const context = await request.newContext({
+    baseURL,
+  })
+
+  // Create initial user
+  await context.post("/api/v2/user", {
+    data: {
+      email,
+      username,
+      password,
+      organization,
+    },
+  })
+}
+
+export default globalSetup

site/e2e/playwright.config.ts

@@ -0,0 +1,27 @@
+import * as path from "path"
+import { PlaywrightTestConfig } from "@playwright/test"
+
+const config: PlaywrightTestConfig = {
+  testDir: "tests",
+  globalSetup: require.resolve("./globalSetup"),
+
+  // Create junit report file for upload to DataDog
+  reporter: [["junit", { outputFile: "test-results/junit.xml" }]],
+
+  use: {
+    baseURL: "http://localhost:3000",
+    video: "retain-on-failure",
+  },
+
+  // `webServer` tells Playwright to launch a test server - more details here:
+  // https://playwright.dev/docs/test-advanced#launching-a-development-web-server-during-the-tests
+  webServer: {
+    // Run the 'coderd' binary directly
+    command: path.join(__dirname, "../../bin/coderd"),
+    port: 3000,
+    timeout: 120 * 10000,
+    reuseExistingServer: false,
+  },
+}
+
+export default config

site/e2e/pom/BasePom.ts

@@ -0,0 +1,17 @@
+import { Page } from "@playwright/test"
+
+export abstract class BasePom {
+  protected readonly baseURL: string | undefined
+  protected readonly path: string
+  protected readonly page: Page
+
+  constructor(baseURL: string | undefined, path: string, page: Page) {
+    this.baseURL = baseURL
+    this.path = path
+    this.page = page
+  }
+
+  get url(): string {
+    return this.baseURL + this.path
+  }
+}

site/e2e/pom/ProjectsPage.ts

@@ -0,0 +1,8 @@
+import { Page } from "@playwright/test"
+import { BasePom } from "./BasePom"
+
+export class ProjectsPage extends BasePom {
+  constructor(baseURL: string | undefined, page: Page) {
+    super(baseURL, "/projects", page)
+  }
+}

site/e2e/pom/SignInPage.ts

@@ -0,0 +1,14 @@
+import { Page } from "@playwright/test"
+import { BasePom } from "./BasePom"
+
+export class SignInPage extends BasePom {
+  constructor(baseURL: string | undefined, page: Page) {
+    super(baseURL, "/login", page)
+  }
+
+  async submitBuiltInAuthentication(email: string, password: string): Promise<void> {
+    await this.page.fill("id=signin-form-inpt-email", email)
+    await this.page.fill("id=signin-form-inpt-password", password)
+    await this.page.click("id=signin-form-submit")
+  }
+}

site/e2e/pom/index.ts

@@ -0,0 +1,2 @@
+export * from "./ProjectsPage"
+export * from "./SignInPage"

site/e2e/tests/login.spec.ts

@@ -0,0 +1,16 @@
+import { test } from "@playwright/test"
+import { ProjectsPage, SignInPage } from "../pom"
+import { email, password } from "../constants"
+
+test("Login takes user to /projects", async ({ baseURL, page }) => {
+  await page.goto(baseURL + "/", { waitUntil: "networkidle" })
+
+  // Log-in with the default credentials we set up in the development server
+  const signInPage = new SignInPage(baseURL, page)
+  await signInPage.submitBuiltInAuthentication(email, password)
+
+  const projectsPage = new ProjectsPage(baseURL, page)
+  await page.waitForNavigation({ url: projectsPage.url, waitUntil: "networkidle" })
+
+  await page.waitForSelector("text=Projects")
+})

site/jest.config.js

@@ -17,7 +17,7 @@ module.exports = {
       },
       testEnvironment: "jsdom",
       testRegex: "(/__tests__/.*|(\\.|/)(test|spec))\\.tsx?$",
-      testPathIgnorePatterns: ["/node_modules/", "/__tests__/fakes"],
+      testPathIgnorePatterns: ["/node_modules/", "/__tests__/fakes", "/e2e/"],
       moduleDirectories: ["node_modules", "<rootDir>"],
     },
     {
@@ -45,6 +45,7 @@ module.exports = {
     "!<rootDir>/api.ts",
     "!<rootDir>/coverage/**/*.*",
     "!<rootDir>/dev.ts",
+    "!<rootDir>/e2e/**/*.*",
     "!<rootDir>/jest-runner.eslint.config.js",
     "!<rootDir>/jest.config.js",
     "!<rootDir>/next-env.d.ts",
@@ -58,7 +59,7 @@ module.exports = {
       "jest-junit",
       {
         suiteName: "Front-end Jest Tests",
-        outputDirectory: "./test_results",
+        outputDirectory: "./test-results",
         outputName: "junit.xml",
       },
     ],

site/package.json

@@ -12,6 +12,9 @@
     "format:write": "prettier --write '**/*.{css,html,js,json,jsx,md,ts,tsx,yaml,yml}' && sql-formatter -l postgresql ./database/query.sql -o ./database/query.sql",
     "lint": "jest --selectProjects lint",
     "lint:fix": "FIX=true yarn lint",
+    "playwright:install": "playwright install",
+    "playwright:install-deps": "playwright install-deps",
+    "playwright:test": "playwright test --config=e2e/playwright.config.ts",
     "storybook": "start-storybook -p 6006 -s ./static",
     "storybook:build": "build-storybook",
     "test": "jest --selectProjects test",
@@ -21,6 +24,7 @@
     "@material-ui/core": "4.9.4",
     "@material-ui/icons": "4.5.1",
     "@material-ui/lab": "4.0.0-alpha.42",
+    "@playwright/test": "1.17.2",
     "@react-theming/storybook-addon": "1.1.5",
     "@storybook/addon-actions": "6.4.19",
     "@storybook/addon-essentials": "6.4.19",