From 051dc02d361fdf576abd7676fb2377fbe11bd9da Mon Sep 17 00:00:00 2001 From: David Sherret Date: Fri, 11 Jul 2025 17:34:20 -0400 Subject: [PATCH 1/2] fix: support importing bytes and text from npm packages --- deno | 2 +- src/rs_lib/Cargo.lock | 21 ++-- src/rs_lib/Cargo.toml | 8 +- src/rs_lib/lib.rs | 98 ++++++++++--------- tests/bytes_and_text/main.test.ts | 9 ++ tests/bytes_and_text_npm/main.test.ts | 41 ++++++++ .../testdata/data_utf8_bom.txt | 1 + tests/bytes_and_text_npm/testdata/deno.json | 5 + tests/bytes_and_text_npm/testdata/main.ts | 5 + .../testdata/node_modules/package/data.txt | 1 + .../node_modules/package/package.json | 1 + .../bytes_and_text_npm/testdata/package.json | 2 + 12 files changed, 131 insertions(+), 63 deletions(-) create mode 100644 tests/bytes_and_text_npm/main.test.ts create mode 100644 tests/bytes_and_text_npm/testdata/data_utf8_bom.txt create mode 100644 tests/bytes_and_text_npm/testdata/deno.json create mode 100644 tests/bytes_and_text_npm/testdata/main.ts create mode 100644 tests/bytes_and_text_npm/testdata/node_modules/package/data.txt create mode 100644 tests/bytes_and_text_npm/testdata/node_modules/package/package.json create mode 100644 tests/bytes_and_text_npm/testdata/package.json diff --git a/deno b/deno index 8464115..1dd0db2 160000 --- a/deno +++ b/deno @@ -1 +1 @@ -Subproject commit 8464115f262359d6172d7bfaba3c7450f52a0877 +Subproject commit 1dd0db2e16f079424c42a4c06a51bc70d6e76df8 diff --git a/src/rs_lib/Cargo.lock b/src/rs_lib/Cargo.lock index 5b22400..fdad4b2 100644 --- a/src/rs_lib/Cargo.lock +++ b/src/rs_lib/Cargo.lock @@ -425,9 +425,9 @@ dependencies = [ [[package]] name = "deno_ast" -version = "0.48.1" +version = "0.48.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ced09fdb8884e29716cc0691e8510f9c655762bbb9da3111dacc0a2ef6e8960" +checksum = "9a4fa191c178c9354d751d2309becde2697d6bde880c8508aae352e0a5ff4fb4" dependencies = [ "base64 0.22.1", "capacity_builder", @@ -737,9 +737,9 @@ dependencies = [ "serde", "serde_json", "sys_traits", - "temp_deno_which", "thiserror", "url", + "which", "winapi", "windows-sys 0.59.0", ] @@ -2632,15 +2632,6 @@ dependencies = [ "xattr", ] -[[package]] -name = "temp_deno_which" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "366c5ccd670145885feb6efd6bbf2478ed236c4c3839046fcc8e2a1a84c51091" -dependencies = [ - "either", -] - [[package]] name = "termcolor" version = "1.4.1" @@ -2960,6 +2951,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "which" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3fabb953106c3c8eea8306e4393700d7657561cb43122571b172bbfb7c7ba1d" + [[package]] name = "winapi" version = "0.3.9" diff --git a/src/rs_lib/Cargo.toml b/src/rs_lib/Cargo.toml index e65cc69..4fc7bc5 100644 --- a/src/rs_lib/Cargo.toml +++ b/src/rs_lib/Cargo.toml @@ -23,7 +23,7 @@ deno_semver = "=0.8.1" url = "2.5" [dependencies.deno_ast] -version = "=0.48.1" +version = "=0.48.2" features = ["transpiling"] [dependencies.deno_cache_dir] @@ -60,7 +60,11 @@ features = ["sync"] [dependencies.sys_traits] version = "=0.1.16" -features = ["real","wasm"] +features = ["real"] + +[target.'cfg(target_arch = "wasm32")'.dependencies.sys_traits] +version = "=0.1.16" +features = ["real", "wasm"] [profile.release] codegen-units = 1 diff --git a/src/rs_lib/lib.rs b/src/rs_lib/lib.rs index 81aa34a..fae6872 100644 --- a/src/rs_lib/lib.rs +++ b/src/rs_lib/lib.rs @@ -40,8 +40,9 @@ use deno_resolver::file_fetcher::PermissionedFileFetcher; use deno_resolver::file_fetcher::PermissionedFileFetcherOptions; use deno_resolver::graph::DefaultDenoResolverRc; use deno_resolver::graph::ResolveWithGraphOptions; -use deno_resolver::loader::PreparedModuleLoader; -use deno_resolver::loader::PreparedModuleOrAsset; +use deno_resolver::loader::LoadCodeSourceErrorKind; +use deno_resolver::loader::LoadedModuleOrAsset; +use deno_resolver::loader::ModuleLoader; use deno_resolver::loader::RequestedModuleType; use deno_resolver::npm::DenoInNpmPackageChecker; use deno_semver::SmallStackString; @@ -293,10 +294,7 @@ impl DenoWorkspace { resolver_factory: self.resolver_factory.clone(), npm_installer_factory: self.npm_installer_factory.clone(), parsed_source_cache: self.resolver_factory.parsed_source_cache().clone(), - prepared_module_loader: self - .resolver_factory - .prepared_module_loader()? - .clone(), + module_loader: self.resolver_factory.module_loader()?.clone(), task_queue: Default::default(), graph: deno_graph::ModuleGraph::new(deno_graph::GraphKind::CodeOnly), }) @@ -314,8 +312,7 @@ pub struct DenoLoader { npm_installer_factory: Arc>, parsed_source_cache: Arc, - prepared_module_loader: - Arc>, + module_loader: Arc>, resolver_factory: Arc>, workspace_factory: Arc>, graph: ModuleGraph, @@ -501,13 +498,12 @@ impl DenoLoader { return Ok(create_external_repsonse(&url)); } - let maybe_prepared_module = self - .prepared_module_loader - .load_prepared_module(&self.graph, &url, requested_module_type) - .await?; - - match maybe_prepared_module { - Some(PreparedModuleOrAsset::Module(m)) => { + match self + .module_loader + .load(&self.graph, &url, None, requested_module_type) + .await + { + Ok(LoadedModuleOrAsset::Module(m)) => { self.parsed_source_cache.free(&m.specifier); Ok(create_module_response( &m.specifier, @@ -515,10 +511,13 @@ impl DenoLoader { m.source.as_bytes(), )) } - Some(PreparedModuleOrAsset::ExternalAsset { specifier }) => { + Ok(LoadedModuleOrAsset::ExternalAsset { + specifier, + statically_analyzable: _, + }) => { let file = self .file_fetcher - .fetch_bypass_permissions(specifier) + .fetch_bypass_permissions(&specifier) .await?; let media_type = MediaType::from_specifier_and_headers( &file.url, @@ -526,42 +525,45 @@ impl DenoLoader { ); Ok(create_module_response(&file.url, media_type, &file.source)) } - None => { - if url.scheme() == "npm" { - bail!( - "Failed resolving '{}'\n\nResolve the npm: specifier to a file: specifier before providing it to the loader.", - url - ) - } - let file = self.file_fetcher.fetch_bypass_permissions(&url).await?; - let media_type = MediaType::from_specifier_and_headers( - &url, - file.maybe_headers.as_ref(), - ); - match requested_module_type { - RequestedModuleType::Text | RequestedModuleType::Bytes => { - Ok(create_module_response(&file.url, media_type, &file.source)) + Err(err) => match err.as_kind() { + LoadCodeSourceErrorKind::LoadUnpreparedModule(_) => { + if url.scheme() == "npm" { + bail!( + "Failed resolving '{}'\n\nResolve the npm: specifier to a file: specifier before providing it to the loader.", + url + ) } - RequestedModuleType::Json - | RequestedModuleType::None - | RequestedModuleType::Other(_) => { - if media_type.is_emittable() { - let str = String::from_utf8_lossy(&file.source); - let value = str.into(); - let source = self - .maybe_transpile(&file.url, media_type, &value, None) - .await?; - Ok(create_module_response( - &file.url, - media_type, - source.as_bytes(), - )) - } else { + let file = self.file_fetcher.fetch_bypass_permissions(&url).await?; + let media_type = MediaType::from_specifier_and_headers( + &url, + file.maybe_headers.as_ref(), + ); + match requested_module_type { + RequestedModuleType::Text | RequestedModuleType::Bytes => { Ok(create_module_response(&file.url, media_type, &file.source)) } + RequestedModuleType::Json + | RequestedModuleType::None + | RequestedModuleType::Other(_) => { + if media_type.is_emittable() { + let str = String::from_utf8_lossy(&file.source); + let value = str.into(); + let source = self + .maybe_transpile(&file.url, media_type, &value, None) + .await?; + Ok(create_module_response( + &file.url, + media_type, + source.as_bytes(), + )) + } else { + Ok(create_module_response(&file.url, media_type, &file.source)) + } + } } } - } + _ => return Err(err.into()), + }, } } diff --git a/tests/bytes_and_text/main.test.ts b/tests/bytes_and_text/main.test.ts index 128ab8c..d9edb46 100644 --- a/tests/bytes_and_text/main.test.ts +++ b/tests/bytes_and_text/main.test.ts @@ -1,3 +1,4 @@ +import { assertEquals } from "@std/assert"; import { assertResponseText, createLoader, @@ -29,4 +30,12 @@ Deno.test("loads jsx transpiled", async () => { await loader.load(dataFileUrl, RequestedModuleType.Text), `Hello there!`, ); + const bytesResponse = await loader.load( + dataFileUrl, + RequestedModuleType.Bytes, + ); + if (bytesResponse.kind !== "module") { + throw new Error("Fail"); + } + assertEquals(bytesResponse.code, Deno.readFileSync(new URL(dataFileUrl))); }); diff --git a/tests/bytes_and_text_npm/main.test.ts b/tests/bytes_and_text_npm/main.test.ts new file mode 100644 index 0000000..a707a2e --- /dev/null +++ b/tests/bytes_and_text_npm/main.test.ts @@ -0,0 +1,41 @@ +import { assertEquals } from "@std/assert"; +import { + assertResponseText, + createLoader, + RequestedModuleType, + ResolutionMode, + type WorkspaceOptions, +} from "../helpers.ts"; + +Deno.test("loads jsx transpiled", async () => { + const mainTs = import.meta.dirname + "/testdata/main.ts"; + const createWorkspace = async (options?: WorkspaceOptions) => { + return await createLoader({ + configPath: import.meta.dirname + "/testdata/deno.json", + ...(options ?? {}), + }, { + entrypoints: [mainTs], + }); + }; + const { loader } = await createWorkspace(); + + const mainTsUrl = loader.resolve(mainTs, undefined, ResolutionMode.Import); + const dataFileUrl = loader.resolve( + "package/data.txt", + mainTsUrl, + ResolutionMode.Import, + ); + + assertResponseText( + await loader.load(dataFileUrl, RequestedModuleType.Text), + `Hello there!`, + ); + const bytesResponse = await loader.load( + dataFileUrl, + RequestedModuleType.Bytes, + ); + if (bytesResponse.kind !== "module") { + throw new Error("Fail"); + } + assertEquals(bytesResponse.code, Deno.readFileSync(new URL(dataFileUrl))); +}); diff --git a/tests/bytes_and_text_npm/testdata/data_utf8_bom.txt b/tests/bytes_and_text_npm/testdata/data_utf8_bom.txt new file mode 100644 index 0000000..cbfa1ee --- /dev/null +++ b/tests/bytes_and_text_npm/testdata/data_utf8_bom.txt @@ -0,0 +1 @@ +Hello there! \ No newline at end of file diff --git a/tests/bytes_and_text_npm/testdata/deno.json b/tests/bytes_and_text_npm/testdata/deno.json new file mode 100644 index 0000000..8af304e --- /dev/null +++ b/tests/bytes_and_text_npm/testdata/deno.json @@ -0,0 +1,5 @@ +{ + "unstable": [ + "raw-imports" + ] +} diff --git a/tests/bytes_and_text_npm/testdata/main.ts b/tests/bytes_and_text_npm/testdata/main.ts new file mode 100644 index 0000000..2aaf294 --- /dev/null +++ b/tests/bytes_and_text_npm/testdata/main.ts @@ -0,0 +1,5 @@ +import dataBytes from "package/data.txt" with { type: "bytes" }; +import dataText from "package/data.txt" with { type: "text" }; + +console.log(dataBytes); +console.log(dataText); diff --git a/tests/bytes_and_text_npm/testdata/node_modules/package/data.txt b/tests/bytes_and_text_npm/testdata/node_modules/package/data.txt new file mode 100644 index 0000000..cd773cd --- /dev/null +++ b/tests/bytes_and_text_npm/testdata/node_modules/package/data.txt @@ -0,0 +1 @@ +Hello there! \ No newline at end of file diff --git a/tests/bytes_and_text_npm/testdata/node_modules/package/package.json b/tests/bytes_and_text_npm/testdata/node_modules/package/package.json new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/tests/bytes_and_text_npm/testdata/node_modules/package/package.json @@ -0,0 +1 @@ +{} diff --git a/tests/bytes_and_text_npm/testdata/package.json b/tests/bytes_and_text_npm/testdata/package.json new file mode 100644 index 0000000..2c63c08 --- /dev/null +++ b/tests/bytes_and_text_npm/testdata/package.json @@ -0,0 +1,2 @@ +{ +} From b707bed724894b13442e96dfa3deadc71d46a78a Mon Sep 17 00:00:00 2001 From: David Sherret Date: Mon, 14 Jul 2025 11:28:22 -0400 Subject: [PATCH 2/2] update --- deno | 2 +- src/rs_lib/Cargo.toml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/deno b/deno index 1dd0db2..489d9bb 160000 --- a/deno +++ b/deno @@ -1 +1 @@ -Subproject commit 1dd0db2e16f079424c42a4c06a51bc70d6e76df8 +Subproject commit 489d9bbdcf382bb9a07ba64a9792de6e84033f1b diff --git a/src/rs_lib/Cargo.toml b/src/rs_lib/Cargo.toml index 4fc7bc5..275d043 100644 --- a/src/rs_lib/Cargo.toml +++ b/src/rs_lib/Cargo.toml @@ -62,9 +62,9 @@ features = ["sync"] version = "=0.1.16" features = ["real"] -[target.'cfg(target_arch = "wasm32")'.dependencies.sys_traits] +[target."cfg(target_arch = \"wasm32\")".dependencies.sys_traits] version = "=0.1.16" -features = ["real", "wasm"] +features = ["real","wasm"] [profile.release] codegen-units = 1