Add basic smoke tests (#2796)

* Add basic smoke tests

* Refactor e2e tests to use Vitest, allow for customising the Wrangler command, and just generally be more resilient

* Remove initial test file

* smoke -> e2e

* Cleanup

* More resilient

* Linting

* Linting

* Linting

* Linting

* regex?

* Address PR feedback

* Address PR comments

* lint

* Add deployments

---------

Co-authored-by: Jacob M-G Evans <jacobmgevans@gmail.com>

packages/wrangler/e2e/deployments.test.ts

@@ -0,0 +1,239 @@
+import crypto from "node:crypto";
+import path from "node:path";
+import { setTimeout } from "node:timers/promises";
+import { fetch } from "undici";
+import { describe, expect, it } from "vitest";
+import { RUN, runIn } from "./helpers/run";
+import { dedent, makeRoot, seed } from "./helpers/setup";
+
+function matchWorkersDev(stdout: string): string {
+	return stdout.match(
+		/https:\/\/smoke-test-worker-.+?\.(.+?\.workers\.dev)/
+	)?.[1] as string;
+}
+
+function matchWhoamiEmail(stdout: string): string {
+	return stdout.match(/associated with the email (.+?@.+?)!/)?.[1] as string;
+}
+
+async function getEmail(root: string) {
+	const { stdout } = await runIn(root)`
+    $ ${RUN} whoami
+  `;
+	return matchWhoamiEmail(stdout);
+}
+
+describe("deployments", async () => {
+	const root = await makeRoot();
+	const workerName = `smoke-test-worker-${crypto
+		.randomBytes(4)
+		.toString("hex")}`;
+	const workerPath = path.join(root, workerName);
+	let workersDev: string | null = null;
+
+	const email = await getEmail(root);
+
+	it("init worker", async () => {
+		const { stdout } = await runIn(root, { [workerName]: "smoke-test-worker" })`
+    $ ${RUN} init ${workerName}
+    `;
+		expect(stdout).toMatchInlineSnapshot(`
+			"Using npm as package manager.
+			✨ Created smoke-test-worker/wrangler.toml
+			? Would you like to use git to manage this Worker?
+			🤖 Using default value in non-interactive context: yes
+			✨ Initialized git repository at smoke-test-worker
+			? No package.json found. Would you like to create one?
+			🤖 Using default value in non-interactive context: yes
+			✨ Created smoke-test-worker/package.json
+			? Would you like to use TypeScript?
+			🤖 Using default value in non-interactive context: yes
+			✨ Created smoke-test-worker/tsconfig.json
+			? Would you like to create a Worker at smoke-test-worker/src/index.ts?
+			🤖 Using default value in non-interactive context: Fetch handler
+			✨ Created smoke-test-worker/src/index.ts
+			? Would you like us to write your first test with Vitest?
+			🤖 Using default value in non-interactive context: yes
+			✨ Created smoke-test-worker/src/index.test.ts
+
+			added (N) packages, and audited (N) packages in (TIMINGS)
+
+			(N) packages are looking for funding
+			  run \`npm fund\` for details
+
+			found 0 vulnerabilities
+			✨ Installed @cloudflare/workers-types, typescript, and vitest into devDependencies
+
+			To start developing your Worker, run \`cd smoke-test-worker && npm start\`
+			To start testing your Worker, run \`npm test\`
+			To publish your Worker to the Internet, run \`npm run deploy\`"
+		`);
+	});
+	it("publish worker", async () => {
+		const {
+			stdout,
+			stderr,
+			raw: { stdout: rawStdout },
+		} = await runIn(workerPath, { [workerName]: "smoke-test-worker" })`
+	  $ ${RUN} publish
+	`;
+		expect(stdout).toMatchInlineSnapshot(`
+			"Total Upload: xx KiB / gzip: xx KiB
+			Uploaded smoke-test-worker (TIMINGS)
+			Published smoke-test-worker (TIMINGS)
+			  https://smoke-test-worker.SUBDOMAIN.workers.dev
+			Current Deployment ID: 00000000-0000-0000-0000-000000000000"
+		`);
+		expect(stderr).toMatchInlineSnapshot('""');
+		workersDev = matchWorkersDev(rawStdout);
+
+		await setTimeout(2_000);
+		await expect(
+			fetch(`https://${workerName}.${workersDev}`).then((r) => r.text())
+		).resolves.toMatchInlineSnapshot('"Hello World!"');
+	});
+
+	it("list 1 deployment", async () => {
+		const { stdout, stderr } = await runIn(workerPath, {
+			[workerName]: "smoke-test-worker",
+			[email]: "person@example.com",
+		})`
+	  $ ${RUN} deployments list
+	`;
+		expect(stdout).toMatchInlineSnapshot(`
+			"🚧\`wrangler deployments\` is a beta command. Please report any issues to https://github.com/cloudflare/workers-sdk/issues/new/choose
+
+
+			Deployment ID: 00000000-0000-0000-0000-000000000000
+			Created on:    TIMESTAMP
+			Author:        person@example.com
+			Source:        Upload from Wrangler 🤠
+			🟩 Active"
+		`);
+		expect(stderr).toMatchInlineSnapshot('""');
+	});
+
+	it("modify & publish worker", async () => {
+		await seed(workerPath, {
+			"src/index.ts": dedent`
+        export default {
+          fetch(request) {
+            return new Response("Updated Worker!")
+          }
+        }`,
+		});
+		const {
+			stdout,
+			stderr,
+			raw: { stdout: rawStdout },
+		} = await runIn(workerPath, { [workerName]: "smoke-test-worker" })`
+	  $ ${RUN} publish
+	`;
+		expect(stdout).toMatchInlineSnapshot(`
+			"Total Upload: xx KiB / gzip: xx KiB
+			Uploaded smoke-test-worker (TIMINGS)
+			Published smoke-test-worker (TIMINGS)
+			  https://smoke-test-worker.SUBDOMAIN.workers.dev
+			Current Deployment ID: 00000000-0000-0000-0000-000000000000"
+		`);
+		expect(stderr).toMatchInlineSnapshot('""');
+		workersDev = matchWorkersDev(rawStdout);
+
+		await setTimeout(10_000);
+		await expect(
+			fetch(`https://${workerName}.${workersDev}`).then((r) => r.text())
+		).resolves.toMatchInlineSnapshot('"Updated Worker!"');
+	});
+
+	it("list 2 deployments", async () => {
+		const { stdout, stderr } = await runIn(workerPath, {
+			[workerName]: "smoke-test-worker",
+			[email]: "person@example.com",
+		})`
+	  $ ${RUN} deployments list
+	`;
+		expect(stdout).toMatchInlineSnapshot(`
+			"🚧\`wrangler deployments\` is a beta command. Please report any issues to https://github.com/cloudflare/workers-sdk/issues/new/choose
+
+
+			Deployment ID: 00000000-0000-0000-0000-000000000000
+			Created on:    TIMESTAMP
+			Author:        person@example.com
+			Source:        Upload from Wrangler 🤠
+
+			Deployment ID: 00000000-0000-0000-0000-000000000000
+			Created on:    TIMESTAMP
+			Author:        person@example.com
+			Source:        Upload from Wrangler 🤠
+			🟩 Active"
+		`);
+		expect(stderr).toMatchInlineSnapshot('""');
+	});
+
+	it("rollback", async () => {
+		const { stdout, stderr } = await runIn(workerPath, {
+			[workerName]: "smoke-test-worker",
+			[email]: "person@example.com",
+		})`
+	  $ ${RUN} rollback --message "A test message"
+	`;
+		expect(stdout).toMatchInlineSnapshot(`
+			"🚧\`wrangler rollback\` is a beta command. Please report any issues to https://github.com/cloudflare/workers-sdk/issues/new/choose
+
+
+			Successfully rolled back to Deployment ID: 00000000-0000-0000-0000-000000000000
+			Current Deployment ID: 00000000-0000-0000-0000-000000000000"
+		`);
+		expect(stderr).toMatchInlineSnapshot('""');
+	});
+
+	it("list deployments", async () => {
+		const { stdout, stderr } = await runIn(workerPath, {
+			[workerName]: "smoke-test-worker",
+			[email]: "person@example.com",
+		})`
+	  $ ${RUN} deployments list
+	`;
+		expect(stdout).toMatchInlineSnapshot(`
+			"🚧\`wrangler deployments\` is a beta command. Please report any issues to https://github.com/cloudflare/workers-sdk/issues/new/choose
+
+
+			Deployment ID: 00000000-0000-0000-0000-000000000000
+			Created on:    TIMESTAMP
+			Author:        person@example.com
+			Source:        Upload from Wrangler 🤠
+
+			Deployment ID: 00000000-0000-0000-0000-000000000000
+			Created on:    TIMESTAMP
+			Author:        person@example.com
+			Source:        Upload from Wrangler 🤠
+
+			Deployment ID: 00000000-0000-0000-0000-000000000000
+			Created on:    TIMESTAMP
+			Author:        person@example.com
+			Source:        Rollback from Wrangler 🤠
+			Rollback from: 00000000-0000-0000-0000-000000000000
+			Message:       A test message
+			🟩 Active"
+		`);
+		expect(stderr).toMatchInlineSnapshot('""');
+	});
+
+	it("delete worker", async () => {
+		const { stdout, stderr } = await runIn(workerPath, {
+			[workerName]: "smoke-test-worker",
+		})`
+	  $ ${RUN} delete
+	  `;
+		expect(stdout).toMatchInlineSnapshot(`
+			"? Are you sure you want to delete smoke-test-worker? This action cannot be undone.
+			🤖 Using default value in non-interactive context: yes
+			Successfully deleted smoke-test-worker"
+		`);
+		expect(stderr).toMatchInlineSnapshot('""');
+		await setTimeout(10_000);
+		await expect(
+			fetch(`https://${workerName}.${workersDev}`).then((r) => r.status)
+		).resolves.toBe(404);
+	});
+});

packages/wrangler/e2e/helpers/normalise.ts

@@ -0,0 +1,120 @@
+import stripAnsi from "strip-ansi";
+export function normalizeOutput(
+	stdout: string,
+	substitutions?: Record<string, string>
+): string {
+	const functions = [
+		npmStripTimings,
+		removeWorkersDev,
+		removeUUID,
+		normalizeErrorMarkers,
+		replaceByte,
+		stripTrailingWhitespace,
+		normalizeSlashes,
+		normalizeTempDirs,
+		stripTimings,
+		removeVersionHeader,
+		stripAnsi,
+		removeTimestamp,
+	];
+	for (const f of functions) {
+		stdout = f(stdout);
+	}
+	if (substitutions) {
+		for (const [from, to] of Object.entries(substitutions)) {
+			stdout = stdout.replaceAll(from, to);
+		}
+	}
+	return stdout;
+}
+function removeWorkersDev(str: string) {
+	return str.replace(
+		/https:\/\/(.+?)\..+?\.workers\.dev/g,
+		"https://$1.SUBDOMAIN.workers.dev"
+	);
+}
+
+function removeTimestamp(str: string) {
+	return str.replace(/\d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\d\.\d+?Z/g, "TIMESTAMP");
+}
+function removeUUID(str: string) {
+	return str.replace(
+		/\w{8}-\w{4}-\w{4}-\w{4}-\w{12}/g,
+		"00000000-0000-0000-0000-000000000000"
+	);
+}
+
+/**
+ * Remove the Wrangler version/update check header
+ */
+function removeVersionHeader(str: string): string {
+	const header = str.match(/----+\n/);
+	if (header !== null && header.index) {
+		return str.slice(header.index + header[0].length);
+	} else {
+		return str;
+	}
+}
+
+/**
+ * Normalize error `X` markers.
+ *
+ * Windows gets a different character.
+ */
+function normalizeErrorMarkers(str: string): string {
+	return str.replaceAll("✘", "X");
+}
+
+/**
+ * Ensure slashes in the `str` are OS file-system agnostic.
+ *
+ * Use this in snapshot tests to be resilient to file-system differences.
+ */
+export function normalizeSlashes(str: string): string {
+	return str.replace(/\\/g, "/");
+}
+
+/**
+ * Strip "timing data" out of the `stdout` string, since this is not always deterministic.
+ *
+ * Use this in snapshot tests to be resilient to slight changes in timing of processing.
+ */
+export function stripTimings(stdout: string): string {
+	return stdout.replace(/\(\d+\.\d+ sec\)/g, "(TIMINGS)");
+}
+
+/**
+ * Strip npm "timing data" out of the `stdout` string, since this is not always deterministic.
+ *
+ * Use this in snapshot tests to be resilient to slight changes in timing of processing.
+ */
+export function npmStripTimings(stdout: string): string {
+	return stdout
+		.replace(
+			/added \d+ packages, and audited \d+ packages in \d+s/,
+			"added (N) packages, and audited (N) packages in (TIMINGS)"
+		)
+		.replace(
+			/\d+ packages are looking for funding/,
+			"(N) packages are looking for funding"
+		);
+}
+
+export function stripTrailingWhitespace(str: string): string {
+	return str.replace(/[^\S\n]+\n/g, "\n");
+}
+
+/**
+ * Removing leading kilobit (tenth of a byte) from test output due to
+ * variation causing every few tests the value to change by ± .01
+ */
+function replaceByte(stdout: string): string {
+	return stdout.replaceAll(/\d+\.\d+ KiB/g, "xx KiB");
+}
+
+/**
+ * Temp directories are created with random names, so we replace all comments temp dirs in them
+ */
+export function normalizeTempDirs(stdout: string): string {
+	return stdout.replaceAll(/\/\/.+\/wrangler-smoke-.+/g, "//tmpdir");
+}

packages/wrangler/e2e/helpers/run.ts

@@ -0,0 +1,28 @@
+import assert from "node:assert";
+import shellac from "shellac";
+import { normalizeOutput } from "./normalise";
+
+assert(
+	process.env.CLOUDFLARE_ACCOUNT_ID,
+	"Please provide a CLOUDFLARE_ACCOUNT_ID as an environment variable"
+);
+
+const CF_ID = `CLOUDFLARE_ACCOUNT_ID=${process.env.CLOUDFLARE_ACCOUNT_ID}`;
+const WRANGLER = process.env.WRANGLER ?? `npx wrangler@beta`;
+
+const RUN = `${CF_ID} ${WRANGLER}`;
+
+function runIn(
+	directory: string,
+	replacers?: Parameters<typeof normalizeOutput>[1]
+) {
+	return async (...p: Parameters<ReturnType<typeof shellac["in"]>>) => {
+		const { stdout, stderr } = await shellac.in(directory)(...p);
+		return {
+			stdout: normalizeOutput(stdout, replacers),
+			stderr: normalizeOutput(stderr, replacers),
+			raw: { stdout, stderr },
+		};
+	};
+}
+export { RUN, runIn };