fix(compile): bundle workers separately under --bundle (#34531)

Before this change, `new Worker(new URL("./w.ts",
import.meta.url), { type: "module" })` in source compiled with
\`--bundle\` failed at runtime: the bundle preserves the URL string
but the original worker source isn't in the VFS, so the worker thread
can't load.

This scans the main bundle output for the
\`new Worker(new URL(..., import.meta.url))\` shape, bundles each
unique target as its own entrypoint (the existing path-rewriter and
CJS-shim detection both run on the worker bundles too), writes each
next to the main bundle as \`.deno_compile_worker_*.mjs\`, and
rewrites the URL literals in the main bundle to point at those files.
The worker bundles are pushed onto \`--include\` so they ship in the
VFS; the \`needs_npm_embed\` signal ORs the workers' results so a
worker with CJS deps still pulls the npm tree along.

Tested with a main + 1 worker exchanging a single message; spec test
lives at \`tests/specs/compile/bundle/worker/\`.

Known limitation: only the exact
\`new Worker(new URL("X", import.meta.url), ...)\` shape is detected.
Computed URLs, workers spawned by transitive dependencies, and
minified bundles (where the pattern is mangled) will still fail with a
path-not-found at runtime.

Author: bartlomieju

cli/tools/compile.rs

@@ -179,25 +179,26 @@ pub async fn compile(
       "{} deno compile --bundle is experimental and may change.",
       colors::yellow("Warning")
     );
-    // Write the bundle next to the working directory rather than the system
-    // temp dir so the embedded VFS path stays relative to the project and
-    // doesn't bake the build machine's temp path into the binary.
-    let initial_cwd = flags.initial_cwd.clone().unwrap_or_else(|| {
-      crate::util::env::resolve_cwd(None).unwrap().to_path_buf()
-    });
-    let bundle_path = initial_cwd.join(format!(
-      ".deno_compile_bundle_{:08x}.mjs",
-      rand::thread_rng().r#gen::<u32>()
-    ));
-    // Register for cleanup before writing so a partially written file is
-    // removed even if bundling fails.
-    let guard = CleanupGuard(vec![bundle_path.clone()]);
-    let needs_npm_embed =
-      run_bundle_for_compile(&flags, &compile_flags, &bundle_path).await?;
+    let BundleForCompileResult {
+      path: bundle_path,
+      needs_npm_embed,
+      extra_cleanup,
+    } = run_bundle_for_compile(&flags, &compile_flags)
+      .boxed_local()
+      .await?;
     flags.internal.compile_bundle_embed_node_modules = needs_npm_embed;
     compile_flags.source_file = bundle_path.to_string_lossy().into_owned();
+    // Make sure any worker bundles travel along in the VFS so the runtime
+    // `new Worker(new URL(..., import.meta.url))` lookup hits them.
+    for worker_path in &extra_cleanup {
+      compile_flags
+        .include
+        .push(worker_path.display().to_string());
+    }
     flags.subcommand = DenoSubcommand::Compile(compile_flags.clone());
-    Some(guard)
+    let mut cleanup = vec![bundle_path];
+    cleanup.extend(extra_cleanup);
+    Some(CleanupGuard(cleanup))
   } else {
     None
   };
@@ -211,49 +212,163 @@ pub async fn compile(
   }
 }
 
-/// Bundles the entrypoint and writes the result to `bundle_path`. Returns
-/// `true` when the bundle needs the npm tree embedded in the binary, i.e.
-/// esbuild's CJS-from-ESM wrapper appears in the output and runtime
-/// require()s against npm package paths will happen.
+struct BundleForCompileResult {
+  path: PathBuf,
+  /// True when esbuild's CJS-from-ESM wrapper appears in the bundle (in the
+  /// main entry or any worker), which means runtime require()s against npm
+  /// package paths will happen. The standalone binary writer reads this to
+  /// decide whether to embed the npm tree.
+  needs_npm_embed: bool,
+  /// Worker bundle files produced alongside the main one; they live next to
+  /// the main bundle and must be cleaned up too.
+  extra_cleanup: Vec<PathBuf>,
+}
+
 async fn run_bundle_for_compile(
   flags: &Flags,
   compile_flags: &CompileFlags,
-  bundle_path: &Path,
-) -> Result<bool, AnyError> {
+) -> Result<BundleForCompileResult, AnyError> {
   let bundle_flags = Arc::new(flags.clone());
+  let initial_cwd = flags.initial_cwd.clone().unwrap_or_else(|| {
+    crate::util::env::resolve_cwd(None).unwrap().to_path_buf()
+  });
 
+  let main_bytes = bundle_one_for_compile(
+    bundle_flags.clone(),
+    compile_flags.source_file.clone(),
+  )
+  .await?;
+  let main_rewrite = rewrite_absolute_bundle_paths(&main_bytes, &initial_cwd)?;
+  let mut needs_npm_embed = main_rewrite.rewrote_paths;
+
+  // Scan the main bundle for `new Worker(new URL("X", import.meta.url))`
+  // patterns. For each unique X, bundle the target as a separate entry,
+  // write it next to the main bundle, and rewrite the URL string in the
+  // main bundle to point at the worker bundle's file name. The worker
+  // bundles are then reachable through `new URL(..., import.meta.url)`
+  // at runtime, just like the main bundle.
+  let main_src = std::str::from_utf8(&main_rewrite.bytes)
+    .context("Bundle output is not valid UTF-8")?;
+  let worker_urls = discover_worker_urls(main_src);
+
+  let mut url_replacements: Vec<(String, String)> = Vec::new();
+  let mut extra_cleanup: Vec<PathBuf> = Vec::new();
+  for worker_url in &worker_urls {
+    let worker_abs = if Path::new(worker_url).is_absolute() {
+      PathBuf::from(worker_url)
+    } else {
+      initial_cwd.join(worker_url)
+    };
+    if !worker_abs.exists() {
+      log::warn!(
+        "{} Worker entrypoint '{}' was not found on disk; leaving it as-is. The compiled binary will fail to start this worker unless the file is provided via --include.",
+        colors::yellow("Warning"),
+        worker_url,
+      );
+      continue;
+    }
+    let worker_bytes = bundle_one_for_compile(
+      bundle_flags.clone(),
+      worker_abs.display().to_string(),
+    )
+    .await?;
+    let worker_rewrite =
+      rewrite_absolute_bundle_paths(&worker_bytes, &initial_cwd)?;
+    needs_npm_embed |= worker_rewrite.rewrote_paths;
+
+    let worker_path = initial_cwd.join(format!(
+      ".deno_compile_worker_{:08x}.mjs",
+      rand::thread_rng().r#gen::<u32>()
+    ));
+    std::fs::write(&worker_path, &worker_rewrite.bytes).with_context(|| {
+      format!(
+        "Writing bundled worker entrypoint to '{}'",
+        worker_path.display()
+      )
+    })?;
+    let worker_file_name = worker_path
+      .file_name()
+      .unwrap()
+      .to_string_lossy()
+      .into_owned();
+    url_replacements
+      .push((worker_url.clone(), format!("./{worker_file_name}")));
+    extra_cleanup.push(worker_path);
+  }
+
+  // Apply URL replacements to the main bundle source.
+  let final_main_src = if url_replacements.is_empty() {
+    main_src.to_string()
+  } else {
+    rewrite_worker_urls(main_src, &url_replacements)
+  };
+
+  let bundle_path = initial_cwd.join(format!(
+    ".deno_compile_bundle_{:08x}.mjs",
+    rand::thread_rng().r#gen::<u32>()
+  ));
+  std::fs::write(&bundle_path, final_main_src.as_bytes()).with_context(
+    || format!("Writing bundled entrypoint to '{}'", bundle_path.display()),
+  )?;
+
+  Ok(BundleForCompileResult {
+    path: bundle_path,
+    needs_npm_embed,
+    extra_cleanup,
+  })
+}
+
+async fn bundle_one_for_compile(
+  flags: Arc<Flags>,
+  entrypoint: String,
+) -> Result<Vec<u8>, AnyError> {
   // Always leave `.node` files external. esbuild has no loader for them
   // and would error if it tried to inline a native binary; with this
   // pattern the require() calls are emitted verbatim and resolved at
   // runtime against the embedded VFS by the native addon loader.
   let external = vec!["*.node".to_string()];
+  super::bundle::bundle_for_compile(flags, entrypoint, external)
+    .boxed_local()
+    .await
+}
 
-  let bytes = super::bundle::bundle_for_compile(
-    bundle_flags,
-    compile_flags.source_file.clone(),
-    external,
-  )
-  .boxed_local()
-  .await?;
+/// Pull the relative path argument out of every
+/// `new Worker(new URL("X", import.meta.url), …)` in the bundle source.
+/// Returns unique paths in source order.
+fn discover_worker_urls(bundle_src: &str) -> Vec<String> {
+  let pattern = lazy_regex::regex!(
+    r#"new\s+Worker\s*\(\s*new\s+URL\s*\(\s*"([^"]+)"\s*,\s*import\.meta\.url"#
+  );
+  let mut seen = std::collections::HashSet::new();
+  let mut out = Vec::new();
+  for caps in pattern.captures_iter(bundle_src) {
+    let url = caps.get(1).unwrap().as_str().to_string();
+    if seen.insert(url.clone()) {
+      out.push(url);
+    }
+  }
+  out
+}
 
-  // The bundle is written next to the working directory, so rewrite paths
-  // relative to that directory.
-  let bundle_dir = bundle_path.parent().unwrap_or_else(|| Path::new("."));
-
-  // esbuild's CJS-from-ESM wrapper hardcodes the absolute on-disk path of
-  // each CJS module it imports. At runtime the embedded VFS is mounted
-  // somewhere else (a temp dir under the user's $TMPDIR), so those paths
-  // miss. Rewrite each such literal to a runtime-relative resolution
-  // against `import.meta.url` so it survives the relocation.
-  let RewriteResult {
-    bytes: rewritten,
-    rewrote_paths,
-  } = rewrite_absolute_bundle_paths(&bytes, bundle_dir)?;
-
-  std::fs::write(bundle_path, &rewritten).with_context(|| {
-    format!("Writing bundled entrypoint to '{}'", bundle_path.display())
-  })?;
-  Ok(rewrote_paths)
+fn rewrite_worker_urls(
+  bundle_src: &str,
+  replacements: &[(String, String)],
+) -> String {
+  let pattern = lazy_regex::regex!(
+    r#"(new\s+Worker\s*\(\s*new\s+URL\s*\(\s*")([^"]+)("\s*,\s*import\.meta\.url)"#
+  );
+  pattern
+    .replace_all(bundle_src, |caps: &regex::Captures<'_>| {
+      let original = caps.get(2).unwrap().as_str();
+      if let Some((_, replacement)) =
+        replacements.iter().find(|(orig, _)| orig == original)
+      {
+        format!("{}{}{}", &caps[1], replacement, &caps[3])
+      } else {
+        caps[0].to_string()
+      }
+    })
+    .into_owned()
 }
 
 struct RewriteResult {

tests/specs/compile/bundle/worker/__test__.jsonc

@@ -0,0 +1,24 @@
+{
+  "tempDir": true,
+  "steps": [{
+    "if": "unix",
+    "args": "compile --bundle --no-check --output app main.ts",
+    "output": "compile.out"
+  }, {
+    "if": "unix",
+    "commandName": "./app",
+    "args": [],
+    "output": "run.out",
+    "exitCode": 0
+  }, {
+    "if": "windows",
+    "args": "compile --bundle --no-check --output app.exe main.ts",
+    "output": "compile.out"
+  }, {
+    "if": "windows",
+    "commandName": "./app.exe",
+    "args": [],
+    "output": "run.out",
+    "exitCode": 0
+  }]
+}

tests/specs/compile/bundle/worker/compile.out

@@ -0,0 +1 @@
+[WILDCARD]deno compile --bundle is experimental[WILDCARD]

tests/specs/compile/bundle/worker/main.ts

@@ -0,0 +1,8 @@
+const w = new Worker(new URL("./worker.ts", import.meta.url), {
+  type: "module",
+});
+w.onmessage = (e) => {
+  console.log("main got:", e.data);
+  w.terminate();
+};
+w.postMessage("ping");

tests/specs/compile/bundle/worker/run.out

@@ -0,0 +1 @@
+main got: pong-ping

tests/specs/compile/bundle/worker/worker.ts

@@ -0,0 +1,3 @@
+self.onmessage = (e) => {
+  (self as unknown as Worker).postMessage("pong-" + e.data);
+};