Skip to content

Commit

Permalink
Implement additional E2E tests for wrangler dev (--remote) (#4486)
Browse files Browse the repository at this point in the history
* Implement additional E2E tests for `wrangler dev (--remote)`

* Rebase + rework e2e to use readUntil

* Kill Wrangler

* Use prebuilt node-pty to speedup CI installation

---------

Co-authored-by: Samuel Macleod <smacleod@cloudflare.com>
  • Loading branch information
mrbbot and penalosa authored May 15, 2024
1 parent abf0a7a commit 82415b5
Show file tree
Hide file tree
Showing 25 changed files with 2,368 additions and 1,945 deletions.
12 changes: 10 additions & 2 deletions .github/workflows/e2e.yml
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ jobs:
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.os }}-${{ matrix.node }}
cancel-in-progress: true
timeout-minutes: 25
timeout-minutes: 40
if: github.repository_owner == 'cloudflare' && (github.event_name != 'pull_request' || (github.event_name == 'pull_request' && contains(github.event.*.labels.*.name, 'e2e' )))
name: "E2E Test"
strategy:
Expand Down Expand Up @@ -55,10 +55,11 @@ jobs:
env:
CLOUDFLARE_API_TOKEN: ${{ secrets.TEST_CLOUDFLARE_API_TOKEN }}
CLOUDFLARE_ACCOUNT_ID: ${{ secrets.TEST_CLOUDFLARE_ACCOUNT_ID }}
WRANGLER: node ${{ github.workspace}}/temp/wrangler/bin/wrangler.js
WRANGLER: node --no-warnings ${{ github.workspace}}/temp/wrangler/bin/wrangler.js
WRANGLER_IMPORT: ${{ github.workspace}}/temp/wrangler/wrangler-dist/cli.js
NODE_OPTIONS: "--max_old_space_size=8192"
WRANGLER_LOG_PATH: ${{ runner.temp }}/wrangler-debug-logs/
TEST_REPORT_PATH: ${{ runner.temp }}/test-report/index.html
CI_OS: ${{ matrix.os }}

- name: Upload debug logs
Expand All @@ -67,3 +68,10 @@ jobs:
with:
name: e2e-test-debug-logs-${{ matrix.os }}-${{ matrix.node }}
path: ${{ runner.temp }}/wrangler-debug-logs/

- name: Upload test report
if: always()
uses: actions/upload-artifact@v3
with:
name: e2e-test-report-${{ matrix.os }}-${{ matrix.node }}
path: ${{ runner.temp }}/test-report/
5 changes: 2 additions & 3 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@

# Created by https://www.toptal.com/developers/gitignore/api/node,macos
# Edit at https://www.toptal.com/developers/gitignore?templates=node,macos

Expand All @@ -11,7 +10,6 @@
# Icon must end with two \r
Icon


# Thumbnails
._*

Expand Down Expand Up @@ -205,7 +203,6 @@ fixtures/remix-pages-app/functions/\[\[path\]\].js
*.drawio.svg.bkp
*.drawio.svg.dtmp


# Vendored npm dependencies
!vendor/*.tgz

Expand All @@ -228,3 +225,5 @@ dist/**
runtime-versions.md

/worker-rust/target/

.e2e-test-report/
3 changes: 3 additions & 0 deletions .vscode/settings.json
Original file line number Diff line number Diff line change
Expand Up @@ -70,5 +70,8 @@
},
"[typescript]": {
"editor.defaultFormatter": "esbenp.prettier-vscode"
},
"[ignore]": {
"editor.defaultFormatter": "foxundermoon.shell-format"
}
}
6 changes: 3 additions & 3 deletions fixtures/interactive-dev-tests/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,11 @@
"test:watch": "vitest"
},
"devDependencies": {
"fixtures-shared": "workspace:*",
"strip-ansi": "^7.1.0",
"undici": "^5.28.3",
"fixtures-shared": "workspace:*"
"undici": "^5.28.3"
},
"optionalDependencies": {
"node-pty": "^1.0.0"
"@cdktf/node-pty-prebuilt-multiarch": "0.10.1-pre.11"
}
}
8 changes: 4 additions & 4 deletions fixtures/interactive-dev-tests/tests/index.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import stripAnsi from "strip-ansi";
import { fetch } from "undici";
import { afterEach, describe as baseDescribe, expect, it } from "vitest";
import { wranglerEntryPath } from "../../shared/src/run-wrangler-long-lived";
import type pty from "node-pty";
import type pty from "@cdktf/node-pty-prebuilt-multiarch";

// These tests are failing with `Error: read EPIPE` on Windows in CI. There's
// still value running them on macOS and Linux.
Expand Down Expand Up @@ -36,13 +36,13 @@ const ptyOptions: pty.IPtyForkOptions = {
cols: 80,
rows: 30,
cwd: pkgRoot,
env: process.env,
env: process.env as Record<string, string>,
};

// Check `node-pty` installed and working correctly, skipping tests if not
let nodePtySupported = true;
try {
const pty = await import("node-pty");
const pty = await import("@cdktf/node-pty-prebuilt-multiarch");
const ptyProcess = pty.spawn(
process.execPath,
["-p", "'ran node'"],
Expand Down Expand Up @@ -104,7 +104,7 @@ async function startWranglerDev(args: string[]) {
let exitResolve: ((code: number) => void) | undefined;
const exitPromise = new Promise<number>((resolve) => (exitResolve = resolve));

const pty = await import("node-pty");
const pty = await import("@cdktf/node-pty-prebuilt-multiarch");
const ptyProcess = pty.spawn(
process.execPath,
[
Expand Down
9 changes: 1 addition & 8 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
"prettify": "prettier . --write --ignore-unknown",
"test": "vitest run --no-file-parallelism && dotenv -- turbo test --filter=wrangler --filter=miniflare --filter=kv-asset-handler --filter=@cloudflare/vitest-pool-workers --filter=@cloudflare/vitest-pool-workers-examples",
"test:ci": "dotenv -- turbo test:ci --concurrency 1",
"test:e2e": "dotenv -- turbo test:e2e --filter=wrangler",
"test:e2e": "dotenv -- turbo test:e2e --log-order=stream --filter=wrangler",
"test:watch": "turbo test:watch",
"type:tests": "dotenv -- turbo type:tests",
"gen:package": "turbo gen package"
Expand Down Expand Up @@ -79,13 +79,6 @@
"toucan-js@3.2.2": "patches/toucan-js@3.2.2.patch",
"@cloudflare/component-listbox@1.10.6": "patches/@cloudflare__component-listbox@1.10.6.patch",
"capnp-ts@0.7.0": "patches/capnp-ts@0.7.0.patch"
},
"packageExtensions": {
"node-pty": {
"optionalDependencies": {
"node-gyp": "^10.0.1"
}
}
}
}
}
73 changes: 73 additions & 0 deletions packages/wrangler/e2e/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
# E2E tests

This folder contains e2e tests for Wrangler. The tests run in CI against a specific Cloudflare account.

You can also run these tests locally, but you'll need access to the `8d783f274e1f82dc46744c297b015a2f` (DevProd Testing) Cloudflare account. Once you have access, generate an API token for the account with the same scopes Wrangler requests.

You can then run the e2e test suite with the following commands (run them in root of the repo):

```sh
pnpm --filter wrangler deploy .tmp/wrangler
rm .tmp/wrangler/templates/tsconfig.json
CLOUDFLARE_ACCOUNT_ID=8d783f274e1f82dc46744c297b015a2f CLOUDFLARE_API_TOKEN=$CLOUDFLARE_TESTING_API_TOKEN WRANGLER="node --no-warnings $PWD/.tmp/wrangler/bin/wrangler.js" WRANGLER_IMPORT="$PWD/.tmp/wrangler/wrangler-dist/cli.js" pnpm --filter wrangler run test:e2e --retry 0
```

> Make sure to replace `$CLOUDFLARE_TESTING_API_TOKEN` with the actual API token you generated, and make sure you've built Wrangler (with `pnpm build`).
## How tests are written

The main thing to keep in mind is that tests should be written with the custom `e2eTest()` test fixture (see https://vitest.dev/guide/test-context.html#test-extend for more details). This is defined in https://github.com/cloudflare/workers-sdk/tree/main/packages/wrangler/e2e/helpers/e2e-wrangler-test.ts, and is used to define tests exactly as you'd use Vitest's default `test()` function:

```ts
e2eTest("can fetch worker", async ({ run, seed }) => {
await seed({
"wrangler.toml": dedent`
name = "worker"
main = "src/index.ts"
compatibility_date = "2023-01-01"
`,
"src/index.ts": dedent/* javascript */ `
export default{
fetch() {
return new Response("hello world");
},
};
`,
"package.json": dedent`
{
"name": "worker",
"version": "0.0.0",
"private": true
}
`,
});
const worker = run("wrangler dev");

const { url } = await waitForReady(worker);

await expect(fetch(url).then((r) => r.text())).resolves.toMatchInlineSnapshot(
'"hello world"'
);
});
```

The above code snippet demonstrates the two main features of the `e2eTest()` fixture: `run()` and `seed()`. The `e2eTest()` fixture also provisions a temporary directory for each test run, which both `seed()` and `run()` use.

### `seed()`

This command allows you to seed the filesystem for a test run. It takes a single argument representing an object mapping from (relative) file paths to file contents.

### `run()`

`run()` is the way to run a Wrangler command. It takes two parameters:

- `cmd: string` This is the command that will be run. It must start with the string `wrangler` (which will be replaced with the path to the actual Wrangler executable, depending on how the e2e tests are being run).
- `options: { debug, env, cwd }`:
- `debug` turns on Wrangler's debug log level, which can be helpful when asserting against Wrangler's output
- `env` sets the environment for the Wrangler command (it defaults to `process.env`)
- `cwd` sets the folder in which the Wrangler command will be run (it defaults to the temporary directory that `e2eTest()` provisions)

Depending on the type of Wrangler command you're running, there are two ways to use `run()`:

1. If the Wrangler command is expected to exit quickly (i.e. `wrangler deploy`), you can await the call to `run()` (i.e. `await run("wrangler deploy")`). This will resolve with the string output resulting from running `cmd` (mixed `stdout` and `stderr`)
2. If the Wrangler command is expected to be long-running, you can instead call `run()` _without_ `await`-ing it. See `e2e/dev.test.ts` for examples of this.
9 changes: 9 additions & 0 deletions packages/wrangler/e2e/__snapshots__/dev.test.ts.snap
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html

exports[`basic js dev: 'wrangler dev --remote' > can modify worker during wrangler dev --remote 1`] = `"Hello World!"`;

exports[`basic js dev: 'wrangler dev --remote' > can modify worker during wrangler dev --remote 2`] = `"Updated Worker! value"`;

exports[`basic js dev: 'wrangler dev' > can modify worker during wrangler dev 1`] = `"Hello World!"`;

exports[`basic js dev: 'wrangler dev' > can modify worker during wrangler dev 2`] = `"Updated Worker! value"`;
91 changes: 22 additions & 69 deletions packages/wrangler/e2e/c3-integration.test.ts
Original file line number Diff line number Diff line change
@@ -1,41 +1,20 @@
import crypto from "node:crypto";
import { existsSync } from "node:fs";
import path from "node:path";
import shellac from "shellac";
import { fetch } from "undici";
import { beforeAll, describe, expect, it } from "vitest";
import { CLOUDFLARE_ACCOUNT_ID } from "./helpers/account-id";
import { normalizeOutput } from "./helpers/normalize";
import { retry } from "./helpers/retry";
import { beforeAll, describe, expect } from "vitest";
import { e2eTest } from "./helpers/e2e-wrangler-test";
import { generateResourceName } from "./helpers/generate-resource-name";
import { makeRoot } from "./helpers/setup";
import { WRANGLER } from "./helpers/wrangler";

function matchWorkersDev(stdout: string): string {
return stdout.match(
/https:\/\/tmp-e2e-wrangler-.+?\.(.+?\.workers\.dev)/
)?.[1] as string;
}

describe("c3 integration", () => {
let workerName: string;
let workerPath: string;
let workersDev: string | null = null;
let runInRoot: typeof shellac;
let runInWorker: typeof shellac;
let root: string;
let c3Packed: string;
let normalize: (str: string) => string;

beforeAll(async () => {
const root = await makeRoot();
runInRoot = shellac.in(root).env(process.env);
workerName = `tmp-e2e-wrangler-${crypto.randomBytes(4).toString("hex")}`;
workerPath = path.join(root, workerName);
runInWorker = shellac.in(workerPath).env(process.env);
normalize = (str) =>
normalizeOutput(str, {
[workerName]: "tmp-e2e-wrangler",
[CLOUDFLARE_ACCOUNT_ID]: "CLOUDFLARE_ACCOUNT_ID",
});
root = await makeRoot();
workerName = generateResourceName("c3");

const pathToC3 = path.resolve(__dirname, "../../create-cloudflare");
const { stdout: version } = await shellac.in(pathToC3)`
Expand All @@ -45,7 +24,7 @@ describe("c3 integration", () => {
c3Packed = path.join(pathToC3, "pack", version);
});

it("init project via c3", async () => {
e2eTest("init project via c3", async ({ run }) => {
const env = {
...process.env,
WRANGLER_C3_COMMAND: `--package ${c3Packed} dlx create-cloudflare`,
Expand All @@ -55,49 +34,23 @@ describe("c3 integration", () => {
GIT_COMMITTER_EMAIL: "test-user@cloudflare.com",
};

await runInRoot.env(env)`$$ ${WRANGLER} init ${workerName} --yes`;
const init = await run(`wrangler init ${workerName} --yes`, {
env,
cwd: root,
});

expect(existsSync(workerPath)).toBe(true);
});
expect(init).toContain("APPLICATION CREATED");

it("deploy the worker", async () => {
const { stdout, stderr } = await runInWorker`$ ${WRANGLER} deploy`;
expect(normalize(stdout)).toMatchInlineSnapshot(`
"Total Upload: xx KiB / gzip: xx KiB
Uploaded tmp-e2e-wrangler (TIMINGS)
Published tmp-e2e-wrangler (TIMINGS)
https://tmp-e2e-wrangler.SUBDOMAIN.workers.dev
Current Deployment ID: 00000000-0000-0000-0000-000000000000
Current Version ID: 00000000-0000-0000-0000-000000000000
Note: Deployment ID has been renamed to Version ID. Deployment ID is present to maintain compatibility with the previous behavior of this command. This output will change in a future version of Wrangler. To learn more visit: https://developers.cloudflare.com/workers/configuration/versions-and-deployments"
`);
expect(stderr).toMatchInlineSnapshot('""');
workersDev = matchWorkersDev(stdout);
const { text } = await retry(
(s) => s.status !== 200,
async () => {
const r = await fetch(`https://${workerName}.${workersDev}`);
return { text: await r.text(), status: r.status };
}
);
expect(text).toMatchInlineSnapshot('"Hello World!"');
expect(existsSync(path.join(root, workerName))).toBe(true);
});

it("delete the worker", async () => {
const { stdout, stderr } = await runInWorker`$$ ${WRANGLER} delete`;
expect(normalize(stdout)).toMatchInlineSnapshot(`
"? Are you sure you want to delete tmp-e2e-wrangler? This action cannot be undone.
🤖 Using fallback value in non-interactive context: yes
Successfully deleted tmp-e2e-wrangler"
`);
expect(stderr).toMatchInlineSnapshot('""');
const { status } = await retry(
(s) => s.status === 200 || s.status === 500,
async () => {
const r = await fetch(`https://${workerName}.${workersDev}`);
return { text: await r.text(), status: r.status };
}
);
expect(status).toBe(404);
});
e2eTest(
"can run `wrangler dev` on generated worker",
async ({ run, waitForReady }) => {
const worker = run(`wrangler dev`, { cwd: path.join(root, workerName) });
const { url } = await waitForReady(worker);
const res = await fetch(url);
expect(await res.text()).toBe("Hello World!");
}
);
});
Loading

0 comments on commit 82415b5

Please sign in to comment.