Support modules with --no-bundle (#2769)

* Support modules with `--no-bundle`

.changeset/beige-eagles-wave.md

@@ -0,0 +1,76 @@
+---
+"wrangler": minor
+"no-bundle-import": patch
+---
+
+feature: Support modules with `--no-bundle`
+
+When the `--no-bundle` flag is set, Wrangler now has support for uploading additional modules alongside the entrypoint. This will allow modules to be imported at runtime on Cloudflare's Edge. This respects Wrangler's [module rules](https://developers.cloudflare.com/workers/wrangler/configuration/#bundling) configuration, which means that only imports of non-JS modules will trigger an upload by default. For instance, the following code will now work with `--no-bundle` (assuming the `example.wasm` file exists at the correct path):
+
+```js
+// index.js
+import wasm from './example.wasm'
+
+export default {
+  async fetch() {
+    await WebAssembly.instantiate(wasm, ...)
+    ...
+  }
+}
+```
+
+For JS modules, it's necessary to specify an additional [module rule](https://developers.cloudflare.com/workers/wrangler/configuration/#bundling) (or rules) in your `wrangler.toml` to configure your modules as ES modules or Common JS modules. For instance, to upload additional JavaScript files as ES modules, add the following module rule to your `wrangler.toml`, which tells Wrangler that all `**/*.js` files are ES modules.
+
+```toml
+rules = [
+  { type = "ESModule", globs = ["**/*.js"]},
+]
+```
+
+If you have Common JS modules, you'd configure Wrangler with a CommonJS rule (the following rule tells Wrangler that all `.cjs` files are Common JS modules):
+
+```toml
+rules = [
+  { type = "CommonJS", globs = ["**/*.cjs"]},
+]
+```
+
+In most projects, adding a single rule will be sufficient. However, for advanced usecases where you're mixing ES modules and Common JS modules, you'll need to use multiple rule definitions. For instance, the following set of rules will match all `.mjs` files as ES modules, all `.cjs` files as Common JS modules, and the `nested/say-hello.js` file as Common JS.
+
+```toml
+rules = [
+  { type = "CommonJS", globs = ["nested/say-hello.js", "**/*.cjs"]},
+  { type = "ESModule", globs = ["**/*.mjs"]}
+]
+```
+
+If multiple rules overlap, Wrangler will log a warning about the duplicate rules, and will discard additional rules that matches a module. For example, the following rule configuration classifies `dep.js` as both a Common JS module and an ES module:
+
+```toml
+rules = [
+  { type = "CommonJS", globs = ["dep.js"]},
+  { type = "ESModule", globs = ["dep.js"]}
+]
+```
+
+Wrangler will treat `dep.js` as a Common JS module, since that was the first rule that matched, and will log the following warning:
+
+```
+▲ [WARNING] Ignoring duplicate module: dep.js (esm)
+```
+
+This also adds a new configuration option to `wrangler.toml`: `base_dir`. Defaulting to the directory of your Worker's main entrypoint, this tells Wrangler where your additional modules are located, and determines the module paths against which your module rule globs are matched.
+
+For instance, given the following directory structure:
+
+```
+- wrangler.toml
+- src/
+  - index.html
+  - vendor/
+    - dependency.js
+  - js/
+    - index.js
+```
+
+If your `wrangler.toml` had `main = "src/js/index.js"`, you would need to set `base_dir = "src"` in order to be able to import `src/vendor/dependency.js` and `src/index.html` from `src/js/index.js`.

fixtures/no-bundle-import/README.md

@@ -0,0 +1,3 @@
+# `no-bundle-import`
+
+This worker exercises the module collection system when `--no-bundle` is specified. It demonstrates dynamic import and static import, as well as module rules whcih treat different JS files as CommonJS or ESModule

fixtures/no-bundle-import/package.json

@@ -0,0 +1,14 @@
+{
+	"name": "no-bundle-import",
+	"version": "0.0.0",
+	"private": true,
+	"scripts": {
+		"deploy": "wrangler publish",
+		"start": "wrangler dev",
+		"test": "vitest"
+	},
+	"devDependencies": {
+		"get-port": "^6.1.2",
+		"wrangler": "^2.10.0"
+	}
+}

fixtures/no-bundle-import/src/data.txt

@@ -0,0 +1 @@
+TEST DATA

fixtures/no-bundle-import/src/dynamic.cjs

@@ -0,0 +1 @@
+module.exports = "cjs-string";

fixtures/no-bundle-import/src/dynamic.js

@@ -0,0 +1 @@
+export default "dynamic";

fixtures/no-bundle-import/src/index.js

@@ -0,0 +1,75 @@
+import { sayHello } from "./say-hello.js";
+import cjs from "./say-hello.cjs";
+
+import { johnSmith } from "./nested/index.js";
+import WASM from "./simple.wasm";
+import nestedWasm from "./nested/simple.wasm";
+
+import text from "./data.txt";
+import binData from "./data.bin";
+export default {
+	async fetch(request, env, ctx) {
+		const url = new URL(request.url);
+		if (url.pathname === "/dynamic") {
+			return new Response(`${(await import("./dynamic.js")).default}`);
+		}
+		if (url.pathname === "/wasm") {
+			return new Response(
+				await new Promise(async (resolve) => {
+					const moduleImport = {
+						imports: {
+							imported_func(arg) {
+								resolve(arg);
+							},
+						},
+					};
+					const module1 = await WebAssembly.instantiate(WASM, moduleImport);
+					module1.exports.exported_func();
+				})
+			);
+		}
+		if (url.pathname === "/wasm-nested") {
+			return new Response(
+				await new Promise(async (resolve) => {
+					const moduleImport = {
+						imports: {
+							imported_func(arg) {
+								resolve("nested" + arg);
+							},
+						},
+					};
+					const m = await WebAssembly.instantiate(nestedWasm, moduleImport);
+					m.exports.exported_func();
+				})
+			);
+		}
+		if (url.pathname === "/wasm-dynamic") {
+			return new Response(
+				`${await (await import("./nested/index.js")).loadWasm()}`
+			);
+		}
+
+		if (url.pathname.startsWith("/lang")) {
+			const language = url.pathname.split("/lang/")[1];
+			return new Response(
+				`${JSON.parse((await import(`./lang/${language}`)).default).hello}`
+			);
+		}
+
+		if (url.pathname === "/txt") {
+			return new Response(text);
+		}
+		if (url.pathname === "/bin") {
+			return new Response(binData);
+		}
+		if (url.pathname === "/cjs") {
+			return new Response(
+				`CJS: ${cjs.sayHello("Jane Smith")} and ${johnSmith}`
+			);
+		}
+		if (url.pathname === "/cjs-loop") {
+			return new Response(`CJS: ${cjs.loop}`);
+		}
+		return new Response(`${sayHello("Jane Smith")} and ${johnSmith}`);
+	},
+};

fixtures/no-bundle-import/src/index.test.ts

@@ -0,0 +1,85 @@
+import path from "path";
+import { describe, expect, test, beforeAll, afterAll } from "vitest";
+import { unstable_dev } from "../../../packages/wrangler/wrangler-dist/cli.js";
+import type { UnstableDevWorker } from "../../../packages/wrangler/wrangler-dist/cli.js";
+
+describe("Worker", () => {
+	let worker: UnstableDevWorker;
+
+	// TODO: Remove this when `workerd` has Windows support
+	if (process.env.RUNNER_OS === "Windows") {
+		test("dummy windows test", () => {
+			expect(process.env.RUNNER_OS).toStrictEqual("Windows");
+		});
+		return;
+	}
+
+	beforeAll(async () => {
+		worker = await unstable_dev(path.resolve(__dirname, "index.js"), {
+			bundle: false,
+			experimental: { experimentalLocal: true },
+		});
+	}, 30_000);
+
+	afterAll(() => worker.stop());
+
+	test("module traversal results in correct response", async () => {
+		const resp = await worker.fetch();
+		const text = await resp.text();
+		expect(text).toMatchInlineSnapshot(
+			`"Hello Jane Smith and Hello John Smith"`
+		);
+	});
+
+	test("module traversal results in correct response for CommonJS", async () => {
+		const resp = await worker.fetch("/cjs");
+		const text = await resp.text();
+		expect(text).toMatchInlineSnapshot(
+			`"CJS: Hello Jane Smith and Hello John Smith"`
+		);
+	});
+
+	test("correct response for CommonJS which imports ESM", async () => {
+		const resp = await worker.fetch("/cjs-loop");
+		const text = await resp.text();
+		expect(text).toMatchInlineSnapshot('"CJS: cjs-string"');
+	});
+
+	test("support for dynamic imports", async () => {
+		const resp = await worker.fetch("/dynamic");
+		const text = await resp.text();
+		expect(text).toMatchInlineSnapshot(`"dynamic"`);
+	});
+
+	test("basic wasm support", async () => {
+		const resp = await worker.fetch("/wasm");
+		const text = await resp.text();
+		expect(text).toMatchInlineSnapshot('"42"');
+	});
+
+	test("resolves wasm import paths relative to root", async () => {
+		const resp = await worker.fetch("/wasm-nested");
+		const text = await resp.text();
+		expect(text).toMatchInlineSnapshot('"nested42"');
+	});
+
+	test("wasm can be imported from a dynamic import", async () => {
+		const resp = await worker.fetch("/wasm-dynamic");
+		const text = await resp.text();
+		expect(text).toMatchInlineSnapshot('"sibling42subdirectory42"');
+	});
+
+	test("text data can be imported", async () => {
+		const resp = await worker.fetch("/txt");
+		const text = await resp.text();
+		expect(text).toMatchInlineSnapshot('"TEST DATA"');
+	});
+
+	test("binary data can be imported", async () => {
+		const resp = await worker.fetch("/bin");
+		const bin = await resp.arrayBuffer();
+		const expected = new Uint8Array(new ArrayBuffer(4));
+		expected.set([0, 1, 2, 10]);
+		expect(new Uint8Array(bin)).toEqual(expected);
+	});
+});

fixtures/no-bundle-import/src/lang/en.json

@@ -0,0 +1,3 @@
+{
+	"hello": "Hello"
+}

fixtures/no-bundle-import/src/lang/fr.json

@@ -0,0 +1,3 @@
+{
+	"hello": "Bonjour"
+}

fixtures/no-bundle-import/src/nested/index.js

@@ -0,0 +1,35 @@
+import { sayHello } from "../say-hello.js";
+import cjs from "./say-hello.js";
+import subWasm from "../simple.wasm";
+import sibWasm from "./simple.wasm";
+export const johnSmith =
+	sayHello("John Smith") === cjs.sayHello("John Smith")
+		? sayHello("John Smith")
+		: false;
+
+export async function loadWasm() {
+	const sibling = await new Promise(async (resolve) => {
+		const moduleImport = {
+			imports: {
+				imported_func(arg) {
+					resolve("sibling" + arg);
+				},
+			},
+		};
+		const m = await WebAssembly.instantiate(sibWasm, moduleImport);
+		m.exports.exported_func();
+	});
+
+	const subdirectory = await new Promise(async (resolve) => {
+		const moduleImport = {
+			imports: {
+				imported_func(arg) {
+					resolve("subdirectory" + arg);
+				},
+			},
+		};
+		const m = await WebAssembly.instantiate(subWasm, moduleImport);
+		m.exports.exported_func();
+	});
+	return sibling + subdirectory;
+}

fixtures/no-bundle-import/src/nested/say-hello.js

@@ -0,0 +1 @@
+module.exports.sayHello = (name) => `Hello ${name}`;

fixtures/no-bundle-import/src/say-hello.cjs

@@ -0,0 +1,3 @@
+module.exports.sayHello = (name) => `Hello ${name}`;
+
+module.exports.loop = require("./dynamic.cjs");

fixtures/no-bundle-import/src/say-hello.js

@@ -0,0 +1 @@
+export const sayHello = (name) => `Hello ${name}`;

fixtures/no-bundle-import/wrangler.toml

@@ -0,0 +1,9 @@
+name = "no-bundle-import"
+main = "src/index.js"
+compatibility_date = "2023-02-20"
+
+rules = [
+  { type = "CommonJS", globs = ["nested/say-hello.js", "**/*.cjs"]},
+  { type = "ESModule", globs = ["**/*.js"]},
+  { type = "Text", globs = ["**/*.json"], fallthrough = true}
+]