diff --git a/Cargo.lock b/Cargo.lock index c1756449394a6..d34f289554bd3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1965,9 +1965,9 @@ checksum = "cb56e1aa765b4b4f3aadfab769793b7087bb03a4ea4920644a6d238e2df5b9ed" [[package]] name = "import_map" -version = "0.8.0" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09ae88504e9128c4c181a0a4726d868d52aa76de270c7fb00c3c40a8f4fbace4" +checksum = "f99e0f89d56c163538ea6bf1f250049669298a26daeee15a9a18f4118cc503f1" dependencies = [ "indexmap", "log", diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 92687c2839d94..9aae5db8fd6f0 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -69,7 +69,7 @@ env_logger = "=0.8.4" eszip = "=0.16.0" fancy-regex = "=0.7.1" http = "=0.2.4" -import_map = "=0.8.0" +import_map = "=0.9.0" jsonc-parser = { version = "=0.19.0", features = ["serde"] } libc = "=0.2.106" log = { version = "=0.4.14", features = ["serde"] } diff --git a/cli/flags.rs b/cli/flags.rs index 6bb03b9933323..ab8153c4ed00b 100644 --- a/cli/flags.rs +++ b/cli/flags.rs @@ -162,6 +162,13 @@ pub struct UpgradeFlags { pub ca_file: Option, } +#[derive(Clone, Debug, PartialEq, Deserialize, Serialize)] +pub struct VendorFlags { + pub specifiers: Vec, + pub output_path: Option, + pub force: bool, +} + #[derive(Clone, Debug, PartialEq, Deserialize, Serialize)] pub enum DenoSubcommand { Bundle(BundleFlags), @@ -182,6 +189,7 @@ pub enum DenoSubcommand { Test(TestFlags), Types, Upgrade(UpgradeFlags), + Vendor(VendorFlags), } impl Default for DenoSubcommand { @@ -481,6 +489,7 @@ pub fn flags_from_vec(args: Vec) -> clap::Result { Some(("lint", m)) => lint_parse(&mut flags, m), Some(("compile", m)) => compile_parse(&mut flags, m), Some(("lsp", m)) => lsp_parse(&mut flags, m), + Some(("vendor", m)) => vendor_parse(&mut flags, m), _ => handle_repl_flags(&mut flags, ReplFlags { eval: None }), } @@ -552,6 +561,7 @@ If the flag is set, restrict these messages to errors.", .subcommand(test_subcommand()) .subcommand(types_subcommand()) .subcommand(upgrade_subcommand()) + .subcommand(vendor_subcommand()) .long_about(DENO_HELP) .after_help(ENV_VARIABLES_HELP) } @@ -1413,6 +1423,52 @@ update to a different location, use the --output flag .arg(ca_file_arg()) } +fn vendor_subcommand<'a>() -> App<'a> { + App::new("vendor") + .about("Vendor remote modules into a local directory") + .long_about( + "Vendor remote modules into a local directory. + +Analyzes the provided modules along with their dependencies, downloads +remote modules to the output directory, and produces an import map that +maps remote specifiers to the downloaded files. + + deno vendor main.ts + deno run --import-map vendor/import_map.json main.ts + +Remote modules and multiple modules may also be specified: + + deno vendor main.ts test.deps.ts https://deno.land/std/path/mod.ts", + ) + .arg( + Arg::new("specifiers") + .takes_value(true) + .multiple_values(true) + .multiple_occurrences(true) + .required(true), + ) + .arg( + Arg::new("output") + .long("output") + .help("The directory to output the vendored modules to") + .takes_value(true), + ) + .arg( + Arg::new("force") + .long("force") + .short('f') + .help( + "Forcefully overwrite conflicting files in existing output directory", + ) + .takes_value(false), + ) + .arg(config_arg()) + .arg(import_map_arg()) + .arg(lock_arg()) + .arg(reload_arg()) + .arg(ca_file_arg()) +} + fn compile_args(app: App) -> App { app .arg(import_map_arg()) @@ -2237,6 +2293,23 @@ fn upgrade_parse(flags: &mut Flags, matches: &clap::ArgMatches) { }); } +fn vendor_parse(flags: &mut Flags, matches: &clap::ArgMatches) { + ca_file_arg_parse(flags, matches); + config_arg_parse(flags, matches); + import_map_arg_parse(flags, matches); + lock_arg_parse(flags, matches); + reload_arg_parse(flags, matches); + + flags.subcommand = DenoSubcommand::Vendor(VendorFlags { + specifiers: matches + .values_of("specifiers") + .map(|p| p.map(ToString::to_string).collect()) + .unwrap_or_default(), + output_path: matches.value_of("output").map(PathBuf::from), + force: matches.is_present("force"), + }); +} + fn compile_args_parse(flags: &mut Flags, matches: &clap::ArgMatches) { import_map_arg_parse(flags, matches); no_remote_arg_parse(flags, matches); @@ -2443,13 +2516,17 @@ fn no_check_arg_parse(flags: &mut Flags, matches: &clap::ArgMatches) { } fn lock_args_parse(flags: &mut Flags, matches: &clap::ArgMatches) { + lock_arg_parse(flags, matches); + if matches.is_present("lock-write") { + flags.lock_write = true; + } +} + +fn lock_arg_parse(flags: &mut Flags, matches: &clap::ArgMatches) { if matches.is_present("lock") { let lockfile = matches.value_of("lock").unwrap(); flags.lock = Some(PathBuf::from(lockfile)); } - if matches.is_present("lock-write") { - flags.lock_write = true; - } } fn config_arg_parse(flags: &mut Flags, matches: &ArgMatches) { @@ -2512,8 +2589,8 @@ mod tests { /// Creates vector of strings, Vec macro_rules! svec { - ($($x:expr),*) => (vec![$($x.to_string()),*]); -} + ($($x:expr),* $(,)?) => (vec![$($x.to_string()),*]); + } #[test] fn global_flags() { @@ -4895,4 +4972,55 @@ mod tests { .contains("error: The following required arguments were not provided:")); assert!(&error_message.contains("--watch=...")); } + + #[test] + fn vendor_minimal() { + let r = flags_from_vec(svec!["deno", "vendor", "mod.ts",]); + assert_eq!( + r.unwrap(), + Flags { + subcommand: DenoSubcommand::Vendor(VendorFlags { + specifiers: svec!["mod.ts"], + force: false, + output_path: None, + }), + ..Flags::default() + } + ); + } + + #[test] + fn vendor_all() { + let r = flags_from_vec(svec![ + "deno", + "vendor", + "--config", + "deno.json", + "--import-map", + "import_map.json", + "--lock", + "lock.json", + "--force", + "--output", + "out_dir", + "--reload", + "mod.ts", + "deps.test.ts", + ]); + assert_eq!( + r.unwrap(), + Flags { + subcommand: DenoSubcommand::Vendor(VendorFlags { + specifiers: svec!["mod.ts", "deps.test.ts"], + force: true, + output_path: Some(PathBuf::from("out_dir")), + }), + config_path: Some("deno.json".to_string()), + import_map_path: Some("import_map.json".to_string()), + lock: Some(PathBuf::from("lock.json")), + reload: true, + ..Flags::default() + } + ); + } } diff --git a/cli/fs_util.rs b/cli/fs_util.rs index fbdcdc81ae9bb..2f10a523f4af7 100644 --- a/cli/fs_util.rs +++ b/cli/fs_util.rs @@ -362,6 +362,34 @@ pub fn path_has_trailing_slash(path: &Path) -> bool { } } +/// Gets a path with the specified file stem suffix. +/// +/// Ex. `file.ts` with suffix `_2` returns `file_2.ts` +pub fn path_with_stem_suffix(path: &Path, suffix: &str) -> PathBuf { + if let Some(file_name) = path.file_name().map(|f| f.to_string_lossy()) { + if let Some(file_stem) = path.file_stem().map(|f| f.to_string_lossy()) { + if let Some(ext) = path.extension().map(|f| f.to_string_lossy()) { + return if file_stem.to_lowercase().ends_with(".d") { + path.with_file_name(format!( + "{}{}.{}.{}", + &file_stem[..file_stem.len() - ".d".len()], + suffix, + // maintain casing + &file_stem[file_stem.len() - "d".len()..], + ext + )) + } else { + path.with_file_name(format!("{}{}.{}", file_stem, suffix, ext)) + }; + } + } + + path.with_file_name(format!("{}{}", file_name, suffix)) + } else { + path.with_file_name(suffix) + } +} + #[cfg(test)] mod tests { use super::*; @@ -730,4 +758,44 @@ mod tests { assert_eq!(result, expected); } } + + #[test] + fn test_path_with_stem_suffix() { + assert_eq!( + path_with_stem_suffix(&PathBuf::from("/"), "_2"), + PathBuf::from("/_2") + ); + assert_eq!( + path_with_stem_suffix(&PathBuf::from("/test"), "_2"), + PathBuf::from("/test_2") + ); + assert_eq!( + path_with_stem_suffix(&PathBuf::from("/test.txt"), "_2"), + PathBuf::from("/test_2.txt") + ); + assert_eq!( + path_with_stem_suffix(&PathBuf::from("/test/subdir"), "_2"), + PathBuf::from("/test/subdir_2") + ); + assert_eq!( + path_with_stem_suffix(&PathBuf::from("/test/subdir.other.txt"), "_2"), + PathBuf::from("/test/subdir.other_2.txt") + ); + assert_eq!( + path_with_stem_suffix(&PathBuf::from("/test.d.ts"), "_2"), + PathBuf::from("/test_2.d.ts") + ); + assert_eq!( + path_with_stem_suffix(&PathBuf::from("/test.D.TS"), "_2"), + PathBuf::from("/test_2.D.TS") + ); + assert_eq!( + path_with_stem_suffix(&PathBuf::from("/test.d.mts"), "_2"), + PathBuf::from("/test_2.d.mts") + ); + assert_eq!( + path_with_stem_suffix(&PathBuf::from("/test.d.cts"), "_2"), + PathBuf::from("/test_2.d.cts") + ); + } } diff --git a/cli/main.rs b/cli/main.rs index ca6b36f0a5250..f8c5d69df05c4 100644 --- a/cli/main.rs +++ b/cli/main.rs @@ -58,6 +58,7 @@ use crate::flags::RunFlags; use crate::flags::TestFlags; use crate::flags::UninstallFlags; use crate::flags::UpgradeFlags; +use crate::flags::VendorFlags; use crate::fmt_errors::PrettyJsError; use crate::graph_util::graph_lock_or_exit; use crate::graph_util::graph_valid; @@ -1290,6 +1291,15 @@ async fn upgrade_command( Ok(0) } +async fn vendor_command( + flags: Flags, + vendor_flags: VendorFlags, +) -> Result { + let ps = ProcState::build(Arc::new(flags)).await?; + tools::vendor::vendor(ps, vendor_flags).await?; + Ok(0) +} + fn init_v8_flags(v8_flags: &[String]) { let v8_flags_includes_help = v8_flags .iter() @@ -1368,6 +1378,9 @@ fn get_subcommand( DenoSubcommand::Upgrade(upgrade_flags) => { upgrade_command(flags, upgrade_flags).boxed_local() } + DenoSubcommand::Vendor(vendor_flags) => { + vendor_command(flags, vendor_flags).boxed_local() + } } } diff --git a/cli/tests/integration/mod.rs b/cli/tests/integration/mod.rs index cc6770472cb96..2a1e69bd1b0a3 100644 --- a/cli/tests/integration/mod.rs +++ b/cli/tests/integration/mod.rs @@ -84,6 +84,8 @@ mod run; mod test; #[path = "upgrade_tests.rs"] mod upgrade; +#[path = "vendor_tests.rs"] +mod vendor; #[path = "watcher_tests.rs"] mod watcher; #[path = "worker_tests.rs"] diff --git a/cli/tests/integration/vendor_tests.rs b/cli/tests/integration/vendor_tests.rs new file mode 100644 index 0000000000000..4aa883a7ebde2 --- /dev/null +++ b/cli/tests/integration/vendor_tests.rs @@ -0,0 +1,372 @@ +// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. + +use deno_core::serde_json; +use deno_core::serde_json::json; +use pretty_assertions::assert_eq; +use std::fs; +use std::path::PathBuf; +use std::process::Stdio; +use tempfile::TempDir; +use test_util as util; +use util::http_server; + +#[test] +fn output_dir_exists() { + let t = TempDir::new().unwrap(); + let vendor_dir = t.path().join("vendor"); + fs::write(t.path().join("mod.ts"), "").unwrap(); + fs::create_dir_all(&vendor_dir).unwrap(); + fs::write(vendor_dir.join("mod.ts"), "").unwrap(); + + let deno = util::deno_cmd() + .current_dir(t.path()) + .env("NO_COLOR", "1") + .arg("vendor") + .arg("mod.ts") + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + let output = deno.wait_with_output().unwrap(); + assert_eq!( + String::from_utf8_lossy(&output.stderr).trim(), + concat!( + "error: Output directory was not empty. Please specify an empty ", + "directory or use --force to ignore this error and potentially ", + "overwrite its contents.", + ), + ); + assert!(!output.status.success()); + + // ensure it errors when using the `--output` arg too + let deno = util::deno_cmd() + .current_dir(t.path()) + .env("NO_COLOR", "1") + .arg("vendor") + .arg("--output") + .arg("vendor") + .arg("mod.ts") + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + let output = deno.wait_with_output().unwrap(); + assert_eq!( + String::from_utf8_lossy(&output.stderr).trim(), + concat!( + "error: Output directory was not empty. Please specify an empty ", + "directory or use --force to ignore this error and potentially ", + "overwrite its contents.", + ), + ); + assert!(!output.status.success()); + + // now use `--force` + let status = util::deno_cmd() + .current_dir(t.path()) + .env("NO_COLOR", "1") + .arg("vendor") + .arg("mod.ts") + .arg("--force") + .spawn() + .unwrap() + .wait() + .unwrap(); + assert!(status.success()); +} + +#[test] +fn import_map_output_dir() { + let t = TempDir::new().unwrap(); + let vendor_dir = t.path().join("vendor"); + fs::write(t.path().join("mod.ts"), "").unwrap(); + fs::create_dir_all(&vendor_dir).unwrap(); + let import_map_path = vendor_dir.join("import_map.json"); + fs::write( + &import_map_path, + "{ \"imports\": { \"https://localhost/\": \"./localhost/\" }}", + ) + .unwrap(); + + let deno = util::deno_cmd() + .current_dir(t.path()) + .env("NO_COLOR", "1") + .arg("vendor") + .arg("--force") + .arg("--import-map") + .arg(import_map_path) + .arg("mod.ts") + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + let output = deno.wait_with_output().unwrap(); + assert_eq!( + String::from_utf8_lossy(&output.stderr).trim(), + "error: Using an import map found in the output directory is not supported.", + ); + assert!(!output.status.success()); +} + +#[test] +fn standard_test() { + let _server = http_server(); + let t = TempDir::new().unwrap(); + let vendor_dir = t.path().join("vendor2"); + fs::write( + t.path().join("my_app.ts"), + "import {Logger} from 'http://localhost:4545/vendor/query_reexport.ts?testing'; new Logger().log('outputted');", + ).unwrap(); + + let deno = util::deno_cmd() + .current_dir(t.path()) + .arg("vendor") + .arg("my_app.ts") + .arg("--output") + .arg("vendor2") + .env("NO_COLOR", "1") + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + let output = deno.wait_with_output().unwrap(); + assert_eq!( + String::from_utf8_lossy(&output.stderr).trim(), + format!( + concat!( + "Download http://localhost:4545/vendor/query_reexport.ts?testing\n", + "Download http://localhost:4545/vendor/logger.ts?test\n", + "{}", + ), + success_text("2 modules", "vendor2", "my_app.ts"), + ) + ); + assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), ""); + assert!(output.status.success()); + + assert!(vendor_dir.exists()); + assert!(!t.path().join("vendor").exists()); + let import_map: serde_json::Value = serde_json::from_str( + &fs::read_to_string(vendor_dir.join("import_map.json")).unwrap(), + ) + .unwrap(); + assert_eq!( + import_map, + json!({ + "imports": { + "http://localhost:4545/": "./localhost_4545/", + "http://localhost:4545/vendor/query_reexport.ts?testing": "./localhost_4545/vendor/query_reexport.ts", + }, + "scopes": { + "./localhost_4545/": { + "./localhost_4545/vendor/logger.ts?test": "./localhost_4545/vendor/logger.ts" + } + } + }), + ); + + // try running the output with `--no-remote` + let deno = util::deno_cmd() + .current_dir(t.path()) + .env("NO_COLOR", "1") + .arg("run") + .arg("--no-remote") + .arg("--no-check") + .arg("--import-map") + .arg("vendor2/import_map.json") + .arg("my_app.ts") + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + let output = deno.wait_with_output().unwrap(); + assert_eq!(String::from_utf8_lossy(&output.stderr).trim(), ""); + assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "outputted"); + assert!(output.status.success()); +} + +#[test] +fn remote_module_test() { + let _server = http_server(); + let t = TempDir::new().unwrap(); + let vendor_dir = t.path().join("vendor"); + + let deno = util::deno_cmd() + .current_dir(t.path()) + .env("NO_COLOR", "1") + .arg("vendor") + .arg("http://localhost:4545/vendor/query_reexport.ts") + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + let output = deno.wait_with_output().unwrap(); + assert_eq!( + String::from_utf8_lossy(&output.stderr).trim(), + format!( + concat!( + "Download http://localhost:4545/vendor/query_reexport.ts\n", + "Download http://localhost:4545/vendor/logger.ts?test\n", + "{}", + ), + success_text("2 modules", "vendor/", "main.ts"), + ) + ); + assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), ""); + assert!(output.status.success()); + assert!(vendor_dir.exists()); + assert!(vendor_dir + .join("localhost_4545/vendor/query_reexport.ts") + .exists()); + assert!(vendor_dir.join("localhost_4545/vendor/logger.ts").exists()); + let import_map: serde_json::Value = serde_json::from_str( + &fs::read_to_string(vendor_dir.join("import_map.json")).unwrap(), + ) + .unwrap(); + assert_eq!( + import_map, + json!({ + "imports": { + "http://localhost:4545/": "./localhost_4545/", + }, + "scopes": { + "./localhost_4545/": { + "./localhost_4545/vendor/logger.ts?test": "./localhost_4545/vendor/logger.ts" + } + } + }), + ); +} + +#[test] +fn existing_import_map() { + let _server = http_server(); + let t = TempDir::new().unwrap(); + let vendor_dir = t.path().join("vendor"); + fs::write( + t.path().join("mod.ts"), + "import {Logger} from 'http://localhost:4545/vendor/logger.ts';", + ) + .unwrap(); + fs::write( + t.path().join("imports.json"), + r#"{ "imports": { "http://localhost:4545/vendor/": "./logger/" } }"#, + ) + .unwrap(); + fs::create_dir(t.path().join("logger")).unwrap(); + fs::write(t.path().join("logger/logger.ts"), "export class Logger {}") + .unwrap(); + + let status = util::deno_cmd() + .current_dir(t.path()) + .arg("vendor") + .arg("mod.ts") + .arg("--import-map") + .arg("imports.json") + .spawn() + .unwrap() + .wait() + .unwrap(); + assert!(status.success()); + // it should not have found any remote dependencies because + // the provided import map mapped it to a local directory + assert!(!vendor_dir.join("import_map.json").exists()); +} + +#[test] +fn dynamic_import() { + let _server = http_server(); + let t = TempDir::new().unwrap(); + let vendor_dir = t.path().join("vendor"); + fs::write( + t.path().join("mod.ts"), + "import {Logger} from 'http://localhost:4545/vendor/dynamic.ts'; new Logger().log('outputted');", + ).unwrap(); + + let status = util::deno_cmd() + .current_dir(t.path()) + .arg("vendor") + .arg("mod.ts") + .spawn() + .unwrap() + .wait() + .unwrap(); + assert!(status.success()); + let import_map: serde_json::Value = serde_json::from_str( + &fs::read_to_string(vendor_dir.join("import_map.json")).unwrap(), + ) + .unwrap(); + assert_eq!( + import_map, + json!({ + "imports": { + "http://localhost:4545/": "./localhost_4545/", + } + }), + ); + + // try running the output with `--no-remote` + let deno = util::deno_cmd() + .current_dir(t.path()) + .env("NO_COLOR", "1") + .arg("run") + .arg("--allow-read=.") + .arg("--no-remote") + .arg("--no-check") + .arg("--import-map") + .arg("vendor/import_map.json") + .arg("mod.ts") + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + let output = deno.wait_with_output().unwrap(); + assert_eq!(String::from_utf8_lossy(&output.stderr).trim(), ""); + assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "outputted"); + assert!(output.status.success()); +} + +#[test] +fn dynamic_non_analyzable_import() { + let _server = http_server(); + let t = TempDir::new().unwrap(); + fs::write( + t.path().join("mod.ts"), + "import {Logger} from 'http://localhost:4545/vendor/dynamic_non_analyzable.ts'; new Logger().log('outputted');", + ).unwrap(); + + let deno = util::deno_cmd() + .current_dir(t.path()) + .env("NO_COLOR", "1") + .arg("vendor") + .arg("--reload") + .arg("mod.ts") + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + let output = deno.wait_with_output().unwrap(); + // todo(https://github.com/denoland/deno_graph/issues/138): it should warn about + // how it couldn't analyze the dynamic import + assert_eq!( + String::from_utf8_lossy(&output.stderr).trim(), + format!( + "Download http://localhost:4545/vendor/dynamic_non_analyzable.ts\n{}", + success_text("1 module", "vendor/", "mod.ts"), + ) + ); + assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), ""); + assert!(output.status.success()); +} + +fn success_text(module_count: &str, dir: &str, entry_point: &str) -> String { + format!( + concat!( + "Vendored {} into {} directory.\n\n", + "To use vendored modules, specify the `--import-map` flag when invoking deno subcommands:\n", + " deno run -A --import-map {} {}" + ), + module_count, + dir, + PathBuf::from(dir).join("import_map.json").display(), + entry_point, + ) +} diff --git a/cli/tests/testdata/vendor/dynamic.ts b/cli/tests/testdata/vendor/dynamic.ts new file mode 100644 index 0000000000000..e2cbb0e59e94a --- /dev/null +++ b/cli/tests/testdata/vendor/dynamic.ts @@ -0,0 +1,3 @@ +const { Logger } = await import("./logger.ts"); + +export { Logger }; diff --git a/cli/tests/testdata/vendor/dynamic_non_analyzable.ts b/cli/tests/testdata/vendor/dynamic_non_analyzable.ts new file mode 100644 index 0000000000000..1847939f647c5 --- /dev/null +++ b/cli/tests/testdata/vendor/dynamic_non_analyzable.ts @@ -0,0 +1,4 @@ +const value = (() => "./logger.ts")(); +const { Logger } = await import(value); + +export { Logger }; diff --git a/cli/tests/testdata/vendor/logger.ts b/cli/tests/testdata/vendor/logger.ts new file mode 100644 index 0000000000000..97f603a48ba30 --- /dev/null +++ b/cli/tests/testdata/vendor/logger.ts @@ -0,0 +1,5 @@ +export class Logger { + log(text: string) { + console.log(text); + } +} diff --git a/cli/tests/testdata/vendor/query_reexport.ts b/cli/tests/testdata/vendor/query_reexport.ts new file mode 100644 index 0000000000000..5dfafb53293f7 --- /dev/null +++ b/cli/tests/testdata/vendor/query_reexport.ts @@ -0,0 +1 @@ +export * from "./logger.ts?test"; diff --git a/cli/tools/mod.rs b/cli/tools/mod.rs index 83c501ddebd0b..ffea76e1d938d 100644 --- a/cli/tools/mod.rs +++ b/cli/tools/mod.rs @@ -9,3 +9,4 @@ pub mod repl; pub mod standalone; pub mod test; pub mod upgrade; +pub mod vendor; diff --git a/cli/tools/vendor/analyze.rs b/cli/tools/vendor/analyze.rs new file mode 100644 index 0000000000000..0639c04876930 --- /dev/null +++ b/cli/tools/vendor/analyze.rs @@ -0,0 +1,113 @@ +// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. + +use deno_ast::swc::ast::ExportDefaultDecl; +use deno_ast::swc::ast::ExportSpecifier; +use deno_ast::swc::ast::ModuleExportName; +use deno_ast::swc::ast::NamedExport; +use deno_ast::swc::ast::Program; +use deno_ast::swc::visit::noop_visit_type; +use deno_ast::swc::visit::Visit; +use deno_ast::swc::visit::VisitWith; +use deno_ast::ParsedSource; + +/// Gets if the parsed source has a default export. +pub fn has_default_export(source: &ParsedSource) -> bool { + let mut visitor = DefaultExportFinder { + has_default_export: false, + }; + let program = source.program(); + let program: &Program = &program; + program.visit_with(&mut visitor); + visitor.has_default_export +} + +struct DefaultExportFinder { + has_default_export: bool, +} + +impl<'a> Visit for DefaultExportFinder { + noop_visit_type!(); + + fn visit_export_default_decl(&mut self, _: &ExportDefaultDecl) { + self.has_default_export = true; + } + + fn visit_named_export(&mut self, named_export: &NamedExport) { + if named_export + .specifiers + .iter() + .any(export_specifier_has_default) + { + self.has_default_export = true; + } + } +} + +fn export_specifier_has_default(s: &ExportSpecifier) -> bool { + match s { + ExportSpecifier::Default(_) => true, + ExportSpecifier::Namespace(_) => false, + ExportSpecifier::Named(named) => { + let export_name = named.exported.as_ref().unwrap_or(&named.orig); + + match export_name { + ModuleExportName::Str(_) => false, + ModuleExportName::Ident(ident) => &*ident.sym == "default", + } + } + } +} + +#[cfg(test)] +mod test { + use deno_ast::MediaType; + use deno_ast::ParseParams; + use deno_ast::ParsedSource; + use deno_ast::SourceTextInfo; + + use super::has_default_export; + + #[test] + fn has_default_when_export_default_decl() { + let parsed_source = parse_module("export default class Class {}"); + assert!(has_default_export(&parsed_source)); + } + + #[test] + fn has_default_when_named_export() { + let parsed_source = parse_module("export {default} from './test.ts';"); + assert!(has_default_export(&parsed_source)); + } + + #[test] + fn has_default_when_named_export_alias() { + let parsed_source = + parse_module("export {test as default} from './test.ts';"); + assert!(has_default_export(&parsed_source)); + } + + #[test] + fn not_has_default_when_named_export_not_exported() { + let parsed_source = + parse_module("export {default as test} from './test.ts';"); + assert!(!has_default_export(&parsed_source)); + } + + #[test] + fn not_has_default_when_not() { + let parsed_source = parse_module("export {test} from './test.ts'; export class Test{} export * from './test';"); + assert!(!has_default_export(&parsed_source)); + } + + fn parse_module(text: &str) -> ParsedSource { + deno_ast::parse_module(ParseParams { + specifier: "file:///mod.ts".to_string(), + capture_tokens: false, + maybe_syntax: None, + media_type: MediaType::TypeScript, + scope_analysis: false, + source: SourceTextInfo::from_string(text.to_string()), + }) + .unwrap() + } +} diff --git a/cli/tools/vendor/build.rs b/cli/tools/vendor/build.rs new file mode 100644 index 0000000000000..58f351dd89ed4 --- /dev/null +++ b/cli/tools/vendor/build.rs @@ -0,0 +1,577 @@ +// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. + +use std::path::Path; + +use deno_core::error::AnyError; +use deno_graph::Module; +use deno_graph::ModuleGraph; +use deno_graph::ModuleKind; + +use super::analyze::has_default_export; +use super::import_map::build_import_map; +use super::mappings::Mappings; +use super::mappings::ProxiedModule; +use super::specifiers::is_remote_specifier; + +/// Allows substituting the environment for testing purposes. +pub trait VendorEnvironment { + fn create_dir_all(&self, dir_path: &Path) -> Result<(), AnyError>; + fn write_file(&self, file_path: &Path, text: &str) -> Result<(), AnyError>; +} + +pub struct RealVendorEnvironment; + +impl VendorEnvironment for RealVendorEnvironment { + fn create_dir_all(&self, dir_path: &Path) -> Result<(), AnyError> { + Ok(std::fs::create_dir_all(dir_path)?) + } + + fn write_file(&self, file_path: &Path, text: &str) -> Result<(), AnyError> { + Ok(std::fs::write(file_path, text)?) + } +} + +/// Vendors remote modules and returns how many were vendored. +pub fn build( + graph: &ModuleGraph, + output_dir: &Path, + environment: &impl VendorEnvironment, +) -> Result { + assert!(output_dir.is_absolute()); + let all_modules = graph.modules(); + let remote_modules = all_modules + .iter() + .filter(|m| is_remote_specifier(&m.specifier)) + .copied() + .collect::>(); + let mappings = + Mappings::from_remote_modules(graph, &remote_modules, output_dir)?; + + // write out all the files + for module in &remote_modules { + let source = match &module.maybe_source { + Some(source) => source, + None => continue, + }; + let local_path = mappings + .proxied_path(&module.specifier) + .unwrap_or_else(|| mappings.local_path(&module.specifier)); + if !matches!(module.kind, ModuleKind::Esm | ModuleKind::Asserted) { + log::warn!( + "Unsupported module kind {:?} for {}", + module.kind, + module.specifier + ); + continue; + } + environment.create_dir_all(local_path.parent().unwrap())?; + environment.write_file(&local_path, source)?; + } + + // write out the proxies + for (specifier, proxied_module) in mappings.proxied_modules() { + let proxy_path = mappings.local_path(specifier); + let module = graph.get(specifier).unwrap(); + let text = build_proxy_module_source(module, proxied_module); + + environment.write_file(&proxy_path, &text)?; + } + + // create the import map + if !mappings.base_specifiers().is_empty() { + let import_map_text = build_import_map(graph, &all_modules, &mappings); + environment + .write_file(&output_dir.join("import_map.json"), &import_map_text)?; + } + + Ok(remote_modules.len()) +} + +fn build_proxy_module_source( + module: &Module, + proxied_module: &ProxiedModule, +) -> String { + let mut text = format!( + "// @deno-types=\"{}\"\n", + proxied_module.declaration_specifier + ); + let relative_specifier = format!( + "./{}", + proxied_module + .output_path + .file_name() + .unwrap() + .to_string_lossy() + ); + + // for simplicity, always include the `export *` statement as it won't error + // even when the module does not contain a named export + text.push_str(&format!("export * from \"{}\";\n", relative_specifier)); + + // add a default export if one exists in the module + if let Some(parsed_source) = module.maybe_parsed_source.as_ref() { + if has_default_export(parsed_source) { + text.push_str(&format!( + "export {{ default }} from \"{}\";\n", + relative_specifier + )); + } + } + + text +} + +#[cfg(test)] +mod test { + use crate::tools::vendor::test::VendorTestBuilder; + use deno_core::serde_json::json; + use pretty_assertions::assert_eq; + + #[tokio::test] + async fn no_remote_modules() { + let mut builder = VendorTestBuilder::with_default_setup(); + let output = builder + .with_loader(|loader| { + loader.add("/mod.ts", ""); + }) + .build() + .await + .unwrap(); + + assert_eq!(output.import_map, None,); + assert_eq!(output.files, vec![],); + } + + #[tokio::test] + async fn local_specifiers_to_remote() { + let mut builder = VendorTestBuilder::with_default_setup(); + let output = builder + .with_loader(|loader| { + loader + .add( + "/mod.ts", + concat!( + r#"import "https://localhost/mod.ts";"#, + r#"import "https://localhost/other.ts?test";"#, + r#"import "https://localhost/redirect.ts";"#, + ), + ) + .add("https://localhost/mod.ts", "export class Mod {}") + .add("https://localhost/other.ts?test", "export class Other {}") + .add_redirect( + "https://localhost/redirect.ts", + "https://localhost/mod.ts", + ); + }) + .build() + .await + .unwrap(); + + assert_eq!( + output.import_map, + Some(json!({ + "imports": { + "https://localhost/": "./localhost/", + "https://localhost/other.ts?test": "./localhost/other.ts", + "https://localhost/redirect.ts": "./localhost/mod.ts", + } + })) + ); + assert_eq!( + output.files, + to_file_vec(&[ + ("/vendor/localhost/mod.ts", "export class Mod {}"), + ("/vendor/localhost/other.ts", "export class Other {}"), + ]), + ); + } + + #[tokio::test] + async fn remote_specifiers() { + let mut builder = VendorTestBuilder::with_default_setup(); + let output = builder + .with_loader(|loader| { + loader + .add( + "/mod.ts", + concat!( + r#"import "https://localhost/mod.ts";"#, + r#"import "https://other/mod.ts";"#, + ), + ) + .add( + "https://localhost/mod.ts", + concat!( + "export * from './other.ts';", + "export * from './redirect.ts';", + "export * from '/absolute.ts';", + ), + ) + .add("https://localhost/other.ts", "export class Other {}") + .add_redirect( + "https://localhost/redirect.ts", + "https://localhost/other.ts", + ) + .add("https://localhost/absolute.ts", "export class Absolute {}") + .add("https://other/mod.ts", "export * from './sub/mod.ts';") + .add( + "https://other/sub/mod.ts", + concat!( + "export * from '../sub2/mod.ts';", + "export * from '../sub2/other?asdf';", + // reference a path on a different origin + "export * from 'https://localhost/other.ts';", + "export * from 'https://localhost/redirect.ts';", + ), + ) + .add("https://other/sub2/mod.ts", "export class Mod {}") + .add_with_headers( + "https://other/sub2/other?asdf", + "export class Other {}", + &[("content-type", "application/javascript")], + ); + }) + .build() + .await + .unwrap(); + + assert_eq!( + output.import_map, + Some(json!({ + "imports": { + "https://localhost/": "./localhost/", + "https://localhost/redirect.ts": "./localhost/other.ts", + "https://other/": "./other/" + }, + "scopes": { + "./localhost/": { + "./localhost/redirect.ts": "./localhost/other.ts", + "/absolute.ts": "./localhost/absolute.ts", + }, + "./other/": { + "./other/sub2/other?asdf": "./other/sub2/other.js" + } + } + })) + ); + assert_eq!( + output.files, + to_file_vec(&[ + ("/vendor/localhost/absolute.ts", "export class Absolute {}"), + ( + "/vendor/localhost/mod.ts", + concat!( + "export * from './other.ts';", + "export * from './redirect.ts';", + "export * from '/absolute.ts';", + ) + ), + ("/vendor/localhost/other.ts", "export class Other {}"), + ("/vendor/other/mod.ts", "export * from './sub/mod.ts';"), + ( + "/vendor/other/sub/mod.ts", + concat!( + "export * from '../sub2/mod.ts';", + "export * from '../sub2/other?asdf';", + "export * from 'https://localhost/other.ts';", + "export * from 'https://localhost/redirect.ts';", + ) + ), + ("/vendor/other/sub2/mod.ts", "export class Mod {}"), + ("/vendor/other/sub2/other.js", "export class Other {}"), + ]), + ); + } + + #[tokio::test] + async fn same_target_filename_specifiers() { + let mut builder = VendorTestBuilder::with_default_setup(); + let output = builder + .with_loader(|loader| { + loader + .add( + "/mod.ts", + concat!( + r#"import "https://localhost/MOD.TS";"#, + r#"import "https://localhost/mod.TS";"#, + r#"import "https://localhost/mod.ts";"#, + r#"import "https://localhost/mod.ts?test";"#, + r#"import "https://localhost/CAPS.TS";"#, + ), + ) + .add("https://localhost/MOD.TS", "export class Mod {}") + .add("https://localhost/mod.TS", "export class Mod2 {}") + .add("https://localhost/mod.ts", "export class Mod3 {}") + .add("https://localhost/mod.ts?test", "export class Mod4 {}") + .add("https://localhost/CAPS.TS", "export class Caps {}"); + }) + .build() + .await + .unwrap(); + + assert_eq!( + output.import_map, + Some(json!({ + "imports": { + "https://localhost/": "./localhost/", + "https://localhost/mod.TS": "./localhost/mod_2.TS", + "https://localhost/mod.ts": "./localhost/mod_3.ts", + "https://localhost/mod.ts?test": "./localhost/mod_4.ts", + } + })) + ); + assert_eq!( + output.files, + to_file_vec(&[ + ("/vendor/localhost/CAPS.TS", "export class Caps {}"), + ("/vendor/localhost/MOD.TS", "export class Mod {}"), + ("/vendor/localhost/mod_2.TS", "export class Mod2 {}"), + ("/vendor/localhost/mod_3.ts", "export class Mod3 {}"), + ("/vendor/localhost/mod_4.ts", "export class Mod4 {}"), + ]), + ); + } + + #[tokio::test] + async fn multiple_entrypoints() { + let mut builder = VendorTestBuilder::with_default_setup(); + let output = builder + .add_entry_point("/test.deps.ts") + .with_loader(|loader| { + loader + .add("/mod.ts", r#"import "https://localhost/mod.ts";"#) + .add( + "/test.deps.ts", + r#"export * from "https://localhost/test.ts";"#, + ) + .add("https://localhost/mod.ts", "export class Mod {}") + .add("https://localhost/test.ts", "export class Test {}"); + }) + .build() + .await + .unwrap(); + + assert_eq!( + output.import_map, + Some(json!({ + "imports": { + "https://localhost/": "./localhost/", + } + })) + ); + assert_eq!( + output.files, + to_file_vec(&[ + ("/vendor/localhost/mod.ts", "export class Mod {}"), + ("/vendor/localhost/test.ts", "export class Test {}"), + ]), + ); + } + + #[tokio::test] + async fn json_module() { + let mut builder = VendorTestBuilder::with_default_setup(); + let output = builder + .with_loader(|loader| { + loader + .add( + "/mod.ts", + r#"import data from "https://localhost/data.json" assert { type: "json" };"#, + ) + .add("https://localhost/data.json", "{}"); + }) + .build() + .await + .unwrap(); + + assert_eq!( + output.import_map, + Some(json!({ + "imports": { + "https://localhost/": "./localhost/" + } + })) + ); + assert_eq!( + output.files, + to_file_vec(&[("/vendor/localhost/data.json", "{}"),]), + ); + } + + #[tokio::test] + async fn data_urls() { + let mut builder = VendorTestBuilder::with_default_setup(); + + let mod_file_text = r#"import * as b from "data:application/typescript,export%20*%20from%20%22https://localhost/mod.ts%22;";"#; + + let output = builder + .with_loader(|loader| { + loader + .add("/mod.ts", &mod_file_text) + .add("https://localhost/mod.ts", "export class Example {}"); + }) + .build() + .await + .unwrap(); + + assert_eq!( + output.import_map, + Some(json!({ + "imports": { + "https://localhost/": "./localhost/" + } + })) + ); + assert_eq!( + output.files, + to_file_vec(&[("/vendor/localhost/mod.ts", "export class Example {}"),]), + ); + } + + #[tokio::test] + async fn x_typescript_types_no_default() { + let mut builder = VendorTestBuilder::with_default_setup(); + let output = builder + .with_loader(|loader| { + loader + .add("/mod.ts", r#"import "https://localhost/mod.js";"#) + .add_with_headers( + "https://localhost/mod.js", + "export class Mod {}", + &[("x-typescript-types", "https://localhost/mod.d.ts")], + ) + .add("https://localhost/mod.d.ts", "export class Mod {}"); + }) + .build() + .await + .unwrap(); + + assert_eq!( + output.import_map, + Some(json!({ + "imports": { + "https://localhost/": "./localhost/" + } + })) + ); + assert_eq!( + output.files, + to_file_vec(&[ + ("/vendor/localhost/mod.d.ts", "export class Mod {}"), + ( + "/vendor/localhost/mod.js", + concat!( + "// @deno-types=\"https://localhost/mod.d.ts\"\n", + "export * from \"./mod.proxied.js\";\n" + ) + ), + ("/vendor/localhost/mod.proxied.js", "export class Mod {}"), + ]), + ); + } + + #[tokio::test] + async fn x_typescript_types_default_export() { + let mut builder = VendorTestBuilder::with_default_setup(); + let output = builder + .with_loader(|loader| { + loader + .add("/mod.ts", r#"import "https://localhost/mod.js";"#) + .add_with_headers( + "https://localhost/mod.js", + "export default class Mod {}", + &[("x-typescript-types", "https://localhost/mod.d.ts")], + ) + .add("https://localhost/mod.d.ts", "export default class Mod {}"); + }) + .build() + .await + .unwrap(); + + assert_eq!( + output.import_map, + Some(json!({ + "imports": { + "https://localhost/": "./localhost/" + } + })) + ); + assert_eq!( + output.files, + to_file_vec(&[ + ("/vendor/localhost/mod.d.ts", "export default class Mod {}"), + ( + "/vendor/localhost/mod.js", + concat!( + "// @deno-types=\"https://localhost/mod.d.ts\"\n", + "export * from \"./mod.proxied.js\";\n", + "export { default } from \"./mod.proxied.js\";\n", + ) + ), + ( + "/vendor/localhost/mod.proxied.js", + "export default class Mod {}" + ), + ]), + ); + } + + #[tokio::test] + async fn subdir() { + let mut builder = VendorTestBuilder::with_default_setup(); + let output = builder + .with_loader(|loader| { + loader + .add( + "/mod.ts", + r#"import "http://localhost:4545/sub/logger/mod.ts?testing";"#, + ) + .add( + "http://localhost:4545/sub/logger/mod.ts?testing", + "export * from './logger.ts?test';", + ) + .add( + "http://localhost:4545/sub/logger/logger.ts?test", + "export class Logger {}", + ); + }) + .build() + .await + .unwrap(); + + assert_eq!( + output.import_map, + Some(json!({ + "imports": { + "http://localhost:4545/": "./localhost_4545/", + "http://localhost:4545/sub/logger/mod.ts?testing": "./localhost_4545/sub/logger/mod.ts", + }, + "scopes": { + "./localhost_4545/": { + "./localhost_4545/sub/logger/logger.ts?test": "./localhost_4545/sub/logger/logger.ts" + } + } + })) + ); + assert_eq!( + output.files, + to_file_vec(&[ + ( + "/vendor/localhost_4545/sub/logger/logger.ts", + "export class Logger {}", + ), + ( + "/vendor/localhost_4545/sub/logger/mod.ts", + "export * from './logger.ts?test';" + ), + ]), + ); + } + + fn to_file_vec(items: &[(&str, &str)]) -> Vec<(String, String)> { + items + .iter() + .map(|(f, t)| (f.to_string(), t.to_string())) + .collect() + } +} diff --git a/cli/tools/vendor/import_map.rs b/cli/tools/vendor/import_map.rs new file mode 100644 index 0000000000000..7e18d56aa5864 --- /dev/null +++ b/cli/tools/vendor/import_map.rs @@ -0,0 +1,285 @@ +// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. + +use std::collections::BTreeMap; + +use deno_ast::LineAndColumnIndex; +use deno_ast::ModuleSpecifier; +use deno_ast::SourceTextInfo; +use deno_core::serde_json; +use deno_graph::Module; +use deno_graph::ModuleGraph; +use deno_graph::Position; +use deno_graph::Range; +use deno_graph::Resolved; +use serde::Serialize; + +use super::mappings::Mappings; +use super::specifiers::is_remote_specifier; +use super::specifiers::is_remote_specifier_text; + +#[derive(Serialize)] +struct SerializableImportMap { + imports: BTreeMap, + #[serde(skip_serializing_if = "BTreeMap::is_empty")] + scopes: BTreeMap>, +} + +struct ImportMapBuilder<'a> { + mappings: &'a Mappings, + imports: ImportsBuilder<'a>, + scopes: BTreeMap>, +} + +impl<'a> ImportMapBuilder<'a> { + pub fn new(mappings: &'a Mappings) -> Self { + ImportMapBuilder { + mappings, + imports: ImportsBuilder::new(mappings), + scopes: Default::default(), + } + } + + pub fn scope( + &mut self, + base_specifier: &ModuleSpecifier, + ) -> &mut ImportsBuilder<'a> { + self + .scopes + .entry( + self + .mappings + .relative_specifier_text(self.mappings.output_dir(), base_specifier), + ) + .or_insert_with(|| ImportsBuilder::new(self.mappings)) + } + + pub fn into_serializable(self) -> SerializableImportMap { + SerializableImportMap { + imports: self.imports.imports, + scopes: self + .scopes + .into_iter() + .map(|(key, value)| (key, value.imports)) + .collect(), + } + } + + pub fn into_file_text(self) -> String { + let mut text = + serde_json::to_string_pretty(&self.into_serializable()).unwrap(); + text.push('\n'); + text + } +} + +struct ImportsBuilder<'a> { + mappings: &'a Mappings, + imports: BTreeMap, +} + +impl<'a> ImportsBuilder<'a> { + pub fn new(mappings: &'a Mappings) -> Self { + Self { + mappings, + imports: Default::default(), + } + } + + pub fn add(&mut self, key: String, specifier: &ModuleSpecifier) { + self.imports.insert( + key, + self + .mappings + .relative_specifier_text(self.mappings.output_dir(), specifier), + ); + } +} + +pub fn build_import_map( + graph: &ModuleGraph, + modules: &[&Module], + mappings: &Mappings, +) -> String { + let mut import_map = ImportMapBuilder::new(mappings); + visit_modules(graph, modules, mappings, &mut import_map); + + for base_specifier in mappings.base_specifiers() { + import_map + .imports + .add(base_specifier.to_string(), base_specifier); + } + + import_map.into_file_text() +} + +fn visit_modules( + graph: &ModuleGraph, + modules: &[&Module], + mappings: &Mappings, + import_map: &mut ImportMapBuilder, +) { + for module in modules { + let text_info = match &module.maybe_parsed_source { + Some(source) => source.source(), + None => continue, + }; + let source_text = match &module.maybe_source { + Some(source) => source, + None => continue, + }; + + for dep in module.dependencies.values() { + visit_maybe_resolved( + &dep.maybe_code, + graph, + import_map, + &module.specifier, + mappings, + text_info, + source_text, + ); + visit_maybe_resolved( + &dep.maybe_type, + graph, + import_map, + &module.specifier, + mappings, + text_info, + source_text, + ); + } + + if let Some((_, maybe_resolved)) = &module.maybe_types_dependency { + visit_maybe_resolved( + maybe_resolved, + graph, + import_map, + &module.specifier, + mappings, + text_info, + source_text, + ); + } + } +} + +fn visit_maybe_resolved( + maybe_resolved: &Resolved, + graph: &ModuleGraph, + import_map: &mut ImportMapBuilder, + referrer: &ModuleSpecifier, + mappings: &Mappings, + text_info: &SourceTextInfo, + source_text: &str, +) { + if let Resolved::Ok { + specifier, range, .. + } = maybe_resolved + { + let text = text_from_range(text_info, source_text, range); + // if the text is empty then it's probably an x-TypeScript-types + if !text.is_empty() { + handle_dep_specifier( + text, specifier, graph, import_map, referrer, mappings, + ); + } + } +} + +fn handle_dep_specifier( + text: &str, + unresolved_specifier: &ModuleSpecifier, + graph: &ModuleGraph, + import_map: &mut ImportMapBuilder, + referrer: &ModuleSpecifier, + mappings: &Mappings, +) { + let specifier = graph.resolve(unresolved_specifier); + // do not handle specifiers pointing at local modules + if !is_remote_specifier(&specifier) { + return; + } + + let base_specifier = mappings.base_specifier(&specifier); + if is_remote_specifier_text(text) { + if !text.starts_with(base_specifier.as_str()) { + panic!("Expected {} to start with {}", text, base_specifier); + } + + let sub_path = &text[base_specifier.as_str().len()..]; + let expected_relative_specifier_text = + mappings.relative_path(base_specifier, &specifier); + if expected_relative_specifier_text == sub_path { + return; + } + + if referrer.origin() == specifier.origin() { + let imports = import_map.scope(base_specifier); + imports.add(sub_path.to_string(), &specifier); + } else { + import_map.imports.add(text.to_string(), &specifier); + } + } else { + let expected_relative_specifier_text = + mappings.relative_specifier_text(referrer, &specifier); + if expected_relative_specifier_text == text { + return; + } + + let key = if text.starts_with("./") || text.starts_with("../") { + // resolve relative specifier key + let mut local_base_specifier = mappings.local_uri(base_specifier); + local_base_specifier.set_query(unresolved_specifier.query()); + local_base_specifier = local_base_specifier + .join(&unresolved_specifier.path()[1..]) + .unwrap_or_else(|_| { + panic!( + "Error joining {} to {}", + unresolved_specifier.path(), + local_base_specifier + ) + }); + local_base_specifier.set_query(unresolved_specifier.query()); + mappings + .relative_specifier_text(mappings.output_dir(), &local_base_specifier) + } else { + // absolute (`/`) or bare specifier should be left as-is + text.to_string() + }; + let imports = import_map.scope(base_specifier); + imports.add(key, &specifier); + } +} + +fn text_from_range<'a>( + text_info: &SourceTextInfo, + text: &'a str, + range: &Range, +) -> &'a str { + let result = &text[byte_range(text_info, range)]; + if result.starts_with('"') || result.starts_with('\'') { + // remove the quotes + &result[1..result.len() - 1] + } else { + result + } +} + +fn byte_range( + text_info: &SourceTextInfo, + range: &Range, +) -> std::ops::Range { + let start = byte_index(text_info, &range.start); + let end = byte_index(text_info, &range.end); + start..end +} + +fn byte_index(text_info: &SourceTextInfo, pos: &Position) -> usize { + // todo(https://github.com/denoland/deno_graph/issues/79): use byte indexes all the way down + text_info + .byte_index(LineAndColumnIndex { + line_index: pos.line, + column_index: pos.character, + }) + .0 as usize +} diff --git a/cli/tools/vendor/mappings.rs b/cli/tools/vendor/mappings.rs new file mode 100644 index 0000000000000..2e85445dcb939 --- /dev/null +++ b/cli/tools/vendor/mappings.rs @@ -0,0 +1,286 @@ +// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. + +use std::collections::HashMap; +use std::collections::HashSet; +use std::path::Path; +use std::path::PathBuf; + +use deno_ast::MediaType; +use deno_ast::ModuleSpecifier; +use deno_core::error::AnyError; +use deno_graph::Module; +use deno_graph::ModuleGraph; +use deno_graph::Position; +use deno_graph::Resolved; + +use crate::fs_util::path_with_stem_suffix; + +use super::specifiers::dir_name_for_root; +use super::specifiers::get_unique_path; +use super::specifiers::make_url_relative; +use super::specifiers::partition_by_root_specifiers; +use super::specifiers::sanitize_filepath; + +pub struct ProxiedModule { + pub output_path: PathBuf, + pub declaration_specifier: ModuleSpecifier, +} + +/// Constructs and holds the remote specifier to local path mappings. +pub struct Mappings { + output_dir: ModuleSpecifier, + mappings: HashMap, + base_specifiers: Vec, + proxies: HashMap, +} + +impl Mappings { + pub fn from_remote_modules( + graph: &ModuleGraph, + remote_modules: &[&Module], + output_dir: &Path, + ) -> Result { + let partitioned_specifiers = + partition_by_root_specifiers(remote_modules.iter().map(|m| &m.specifier)); + let mut mapped_paths = HashSet::new(); + let mut mappings = HashMap::new(); + let mut proxies = HashMap::new(); + let mut base_specifiers = Vec::new(); + + for (root, specifiers) in partitioned_specifiers.into_iter() { + let base_dir = get_unique_path( + output_dir.join(dir_name_for_root(&root)), + &mut mapped_paths, + ); + for specifier in specifiers { + let media_type = graph.get(&specifier).unwrap().media_type; + let sub_path = sanitize_filepath(&make_url_relative(&root, &{ + let mut specifier = specifier.clone(); + specifier.set_query(None); + specifier + })?); + let new_path = path_with_extension( + &base_dir.join(if cfg!(windows) { + sub_path.replace('/', "\\") + } else { + sub_path + }), + &media_type.as_ts_extension()[1..], + ); + mappings + .insert(specifier, get_unique_path(new_path, &mut mapped_paths)); + } + base_specifiers.push(root.clone()); + mappings.insert(root, base_dir); + } + + // resolve all the "proxy" paths to use for when an x-typescript-types header is specified + for module in remote_modules { + if let Some(( + _, + Resolved::Ok { + specifier, range, .. + }, + )) = &module.maybe_types_dependency + { + // hack to tell if it's an x-typescript-types header + let is_ts_types_header = + range.start == Position::zeroed() && range.end == Position::zeroed(); + if is_ts_types_header { + let module_path = mappings.get(&module.specifier).unwrap(); + let proxied_path = get_unique_path( + path_with_stem_suffix(module_path, ".proxied"), + &mut mapped_paths, + ); + proxies.insert( + module.specifier.clone(), + ProxiedModule { + output_path: proxied_path, + declaration_specifier: specifier.clone(), + }, + ); + } + } + } + + Ok(Self { + output_dir: ModuleSpecifier::from_directory_path(output_dir).unwrap(), + mappings, + base_specifiers, + proxies, + }) + } + + pub fn output_dir(&self) -> &ModuleSpecifier { + &self.output_dir + } + + pub fn local_uri(&self, specifier: &ModuleSpecifier) -> ModuleSpecifier { + if specifier.scheme() == "file" { + specifier.clone() + } else { + let local_path = self.local_path(specifier); + if specifier.path().ends_with('/') { + ModuleSpecifier::from_directory_path(&local_path) + } else { + ModuleSpecifier::from_file_path(&local_path) + } + .unwrap_or_else(|_| { + panic!("Could not convert {} to uri.", local_path.display()) + }) + } + } + + pub fn local_path(&self, specifier: &ModuleSpecifier) -> PathBuf { + if specifier.scheme() == "file" { + specifier.to_file_path().unwrap() + } else { + self + .mappings + .get(specifier) + .as_ref() + .unwrap_or_else(|| { + panic!("Could not find local path for {}", specifier) + }) + .to_path_buf() + } + } + + pub fn relative_path( + &self, + from: &ModuleSpecifier, + to: &ModuleSpecifier, + ) -> String { + let mut from = self.local_uri(from); + let to = self.local_uri(to); + + // workaround using parent directory until https://github.com/servo/rust-url/pull/754 is merged + if !from.path().ends_with('/') { + let local_path = self.local_path(&from); + from = ModuleSpecifier::from_directory_path(local_path.parent().unwrap()) + .unwrap(); + } + + // workaround for url crate not adding a trailing slash for a directory + // it seems to be fixed once a version greater than 2.2.2 is released + let is_dir = to.path().ends_with('/'); + let mut text = from.make_relative(&to).unwrap(); + if is_dir && !text.ends_with('/') && to.query().is_none() { + text.push('/'); + } + text + } + + pub fn relative_specifier_text( + &self, + from: &ModuleSpecifier, + to: &ModuleSpecifier, + ) -> String { + let relative_path = self.relative_path(from, to); + + if relative_path.starts_with("../") || relative_path.starts_with("./") { + relative_path + } else { + format!("./{}", relative_path) + } + } + + pub fn base_specifiers(&self) -> &Vec { + &self.base_specifiers + } + + pub fn base_specifier( + &self, + child_specifier: &ModuleSpecifier, + ) -> &ModuleSpecifier { + self + .base_specifiers + .iter() + .find(|s| child_specifier.as_str().starts_with(s.as_str())) + .unwrap_or_else(|| { + panic!("Could not find base specifier for {}", child_specifier) + }) + } + + pub fn proxied_path(&self, specifier: &ModuleSpecifier) -> Option { + self.proxies.get(specifier).map(|s| s.output_path.clone()) + } + + pub fn proxied_modules( + &self, + ) -> std::collections::hash_map::Iter<'_, ModuleSpecifier, ProxiedModule> { + self.proxies.iter() + } +} + +fn path_with_extension(path: &Path, new_ext: &str) -> PathBuf { + if let Some(file_stem) = path.file_stem().map(|f| f.to_string_lossy()) { + if let Some(old_ext) = path.extension().map(|f| f.to_string_lossy()) { + if file_stem.to_lowercase().ends_with(".d") { + if new_ext.to_lowercase() == format!("d.{}", old_ext.to_lowercase()) { + // maintain casing + return path.to_path_buf(); + } + return path.with_file_name(format!( + "{}.{}", + &file_stem[..file_stem.len() - ".d".len()], + new_ext + )); + } + if new_ext.to_lowercase() == old_ext.to_lowercase() { + // maintain casing + return path.to_path_buf(); + } + let media_type: MediaType = path.into(); + if media_type == MediaType::Unknown { + return path.with_file_name(format!( + "{}.{}", + path.file_name().unwrap().to_string_lossy(), + new_ext + )); + } + } + } + path.with_extension(new_ext) +} + +#[cfg(test)] +mod test { + use pretty_assertions::assert_eq; + + use super::*; + + #[test] + fn test_path_with_extension() { + assert_eq!( + path_with_extension(&PathBuf::from("/test.D.TS"), "ts"), + PathBuf::from("/test.ts") + ); + assert_eq!( + path_with_extension(&PathBuf::from("/test.D.MTS"), "js"), + PathBuf::from("/test.js") + ); + assert_eq!( + path_with_extension(&PathBuf::from("/test.D.TS"), "d.ts"), + // maintains casing + PathBuf::from("/test.D.TS"), + ); + assert_eq!( + path_with_extension(&PathBuf::from("/test.TS"), "ts"), + // maintains casing + PathBuf::from("/test.TS"), + ); + assert_eq!( + path_with_extension(&PathBuf::from("/test.ts"), "js"), + PathBuf::from("/test.js") + ); + assert_eq!( + path_with_extension(&PathBuf::from("/test.js"), "js"), + PathBuf::from("/test.js") + ); + assert_eq!( + path_with_extension(&PathBuf::from("/chai@1.2.3"), "js"), + PathBuf::from("/chai@1.2.3.js") + ); + } +} diff --git a/cli/tools/vendor/mod.rs b/cli/tools/vendor/mod.rs new file mode 100644 index 0000000000000..eb9c91071bc01 --- /dev/null +++ b/cli/tools/vendor/mod.rs @@ -0,0 +1,172 @@ +// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. + +use std::path::Path; +use std::path::PathBuf; + +use deno_core::anyhow::bail; +use deno_core::anyhow::Context; +use deno_core::error::AnyError; +use deno_core::resolve_url_or_path; +use deno_runtime::permissions::Permissions; + +use crate::flags::VendorFlags; +use crate::fs_util; +use crate::lockfile; +use crate::proc_state::ProcState; +use crate::resolver::ImportMapResolver; +use crate::resolver::JsxResolver; +use crate::tools::vendor::specifiers::is_remote_specifier_text; + +mod analyze; +mod build; +mod import_map; +mod mappings; +mod specifiers; +#[cfg(test)] +mod test; + +pub async fn vendor(ps: ProcState, flags: VendorFlags) -> Result<(), AnyError> { + let raw_output_dir = match &flags.output_path { + Some(output_path) => output_path.to_owned(), + None => PathBuf::from("vendor/"), + }; + let output_dir = fs_util::resolve_from_cwd(&raw_output_dir)?; + validate_output_dir(&output_dir, &flags, &ps)?; + let graph = create_graph(&ps, &flags).await?; + let vendored_count = + build::build(&graph, &output_dir, &build::RealVendorEnvironment)?; + + eprintln!( + r#"Vendored {} {} into {} directory. + +To use vendored modules, specify the `--import-map` flag when invoking deno subcommands: + deno run -A --import-map {} {}"#, + vendored_count, + if vendored_count == 1 { + "module" + } else { + "modules" + }, + raw_output_dir.display(), + raw_output_dir.join("import_map.json").display(), + flags + .specifiers + .iter() + .map(|s| s.as_str()) + .find(|s| !is_remote_specifier_text(s)) + .unwrap_or("main.ts"), + ); + + Ok(()) +} + +fn validate_output_dir( + output_dir: &Path, + flags: &VendorFlags, + ps: &ProcState, +) -> Result<(), AnyError> { + if !flags.force && !is_dir_empty(output_dir)? { + bail!(concat!( + "Output directory was not empty. Please specify an empty directory or use ", + "--force to ignore this error and potentially overwrite its contents.", + )); + } + + // check the import map + if let Some(import_map_path) = ps + .maybe_import_map + .as_ref() + .and_then(|m| m.base_url().to_file_path().ok()) + .and_then(|p| fs_util::canonicalize_path(&p).ok()) + { + // make the output directory in order to canonicalize it for the check below + std::fs::create_dir_all(&output_dir)?; + let output_dir = + fs_util::canonicalize_path(output_dir).with_context(|| { + format!("Failed to canonicalize: {}", output_dir.display()) + })?; + + if import_map_path.starts_with(&output_dir) { + // We don't allow using the output directory to help generate the new state + // of itself because supporting this scenario adds a lot of complexity. + bail!( + "Using an import map found in the output directory is not supported." + ); + } + } + + Ok(()) +} + +fn is_dir_empty(dir_path: &Path) -> Result { + match std::fs::read_dir(&dir_path) { + Ok(mut dir) => Ok(dir.next().is_none()), + Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(true), + Err(err) => { + bail!("Error reading directory {}: {}", dir_path.display(), err) + } + } +} + +async fn create_graph( + ps: &ProcState, + flags: &VendorFlags, +) -> Result { + let entry_points = flags + .specifiers + .iter() + .map(|p| { + let url = resolve_url_or_path(p)?; + Ok((url, deno_graph::ModuleKind::Esm)) + }) + .collect::, AnyError>>()?; + + // todo(dsherret): there is a lot of copy and paste here from + // other parts of the codebase. We should consolidate this. + let mut cache = crate::cache::FetchCacher::new( + ps.dir.gen_cache.clone(), + ps.file_fetcher.clone(), + Permissions::allow_all(), + Permissions::allow_all(), + ); + let maybe_locker = lockfile::as_maybe_locker(ps.lockfile.clone()); + let maybe_imports = if let Some(config_file) = &ps.maybe_config_file { + config_file.to_maybe_imports()? + } else { + None + }; + let maybe_import_map_resolver = + ps.maybe_import_map.clone().map(ImportMapResolver::new); + let maybe_jsx_resolver = ps + .maybe_config_file + .as_ref() + .map(|cf| { + cf.to_maybe_jsx_import_source_module() + .map(|im| JsxResolver::new(im, maybe_import_map_resolver.clone())) + }) + .flatten(); + let maybe_resolver = if maybe_jsx_resolver.is_some() { + maybe_jsx_resolver.as_ref().map(|jr| jr.as_resolver()) + } else { + maybe_import_map_resolver + .as_ref() + .map(|im| im.as_resolver()) + }; + + let graph = deno_graph::create_graph( + entry_points, + false, + maybe_imports, + &mut cache, + maybe_resolver, + maybe_locker, + None, + None, + ) + .await; + + graph.lock()?; + graph.valid()?; + + Ok(graph) +} diff --git a/cli/tools/vendor/specifiers.rs b/cli/tools/vendor/specifiers.rs new file mode 100644 index 0000000000000..b869e989c0b7a --- /dev/null +++ b/cli/tools/vendor/specifiers.rs @@ -0,0 +1,251 @@ +// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. + +use std::collections::BTreeMap; +use std::collections::HashSet; +use std::path::PathBuf; + +use deno_ast::ModuleSpecifier; +use deno_core::anyhow::anyhow; +use deno_core::error::AnyError; + +use crate::fs_util::path_with_stem_suffix; + +/// Partitions the provided specifiers by the non-path and non-query parts of a specifier. +pub fn partition_by_root_specifiers<'a>( + specifiers: impl Iterator, +) -> BTreeMap> { + let mut root_specifiers: BTreeMap> = + Default::default(); + for remote_specifier in specifiers { + let mut root_specifier = remote_specifier.clone(); + root_specifier.set_query(None); + root_specifier.set_path("/"); + + let specifiers = root_specifiers.entry(root_specifier).or_default(); + specifiers.push(remote_specifier.clone()); + } + root_specifiers +} + +/// Gets the directory name to use for the provided root. +pub fn dir_name_for_root(root: &ModuleSpecifier) -> PathBuf { + let mut result = String::new(); + if let Some(domain) = root.domain() { + result.push_str(&sanitize_segment(domain)); + } + if let Some(port) = root.port() { + if !result.is_empty() { + result.push('_'); + } + result.push_str(&port.to_string()); + } + let mut result = PathBuf::from(result); + if let Some(segments) = root.path_segments() { + for segment in segments.filter(|s| !s.is_empty()) { + result = result.join(sanitize_segment(segment)); + } + } + + result +} + +/// Gets a unique file path given the provided file path +/// and the set of existing file paths. Inserts to the +/// set when finding a unique path. +pub fn get_unique_path( + mut path: PathBuf, + unique_set: &mut HashSet, +) -> PathBuf { + let original_path = path.clone(); + let mut count = 2; + // case insensitive comparison so the output works on case insensitive file systems + while !unique_set.insert(path.to_string_lossy().to_lowercase()) { + path = path_with_stem_suffix(&original_path, &format!("_{}", count)); + count += 1; + } + path +} + +pub fn make_url_relative( + root: &ModuleSpecifier, + url: &ModuleSpecifier, +) -> Result { + root.make_relative(url).ok_or_else(|| { + anyhow!( + "Error making url ({}) relative to root: {}", + url.to_string(), + root.to_string() + ) + }) +} + +pub fn is_remote_specifier(specifier: &ModuleSpecifier) -> bool { + specifier.scheme().to_lowercase().starts_with("http") +} + +pub fn is_remote_specifier_text(text: &str) -> bool { + text.trim_start().to_lowercase().starts_with("http") +} + +pub fn sanitize_filepath(text: &str) -> String { + text + .chars() + .map(|c| if is_banned_path_char(c) { '_' } else { c }) + .collect() +} + +fn is_banned_path_char(c: char) -> bool { + matches!(c, '<' | '>' | ':' | '"' | '|' | '?' | '*') +} + +fn sanitize_segment(text: &str) -> String { + text + .chars() + .map(|c| if is_banned_segment_char(c) { '_' } else { c }) + .collect() +} + +fn is_banned_segment_char(c: char) -> bool { + matches!(c, '/' | '\\') || is_banned_path_char(c) +} + +#[cfg(test)] +mod test { + use super::*; + use pretty_assertions::assert_eq; + + #[test] + fn partition_by_root_specifiers_same_sub_folder() { + run_partition_by_root_specifiers_test( + vec![ + "https://deno.land/x/mod/A.ts", + "https://deno.land/x/mod/other/A.ts", + ], + vec![( + "https://deno.land/", + vec![ + "https://deno.land/x/mod/A.ts", + "https://deno.land/x/mod/other/A.ts", + ], + )], + ); + } + + #[test] + fn partition_by_root_specifiers_different_sub_folder() { + run_partition_by_root_specifiers_test( + vec![ + "https://deno.land/x/mod/A.ts", + "https://deno.land/x/other/A.ts", + ], + vec![( + "https://deno.land/", + vec![ + "https://deno.land/x/mod/A.ts", + "https://deno.land/x/other/A.ts", + ], + )], + ); + } + + #[test] + fn partition_by_root_specifiers_different_hosts() { + run_partition_by_root_specifiers_test( + vec![ + "https://deno.land/mod/A.ts", + "http://deno.land/B.ts", + "https://deno.land:8080/C.ts", + "https://localhost/mod/A.ts", + "https://other/A.ts", + ], + vec![ + ("http://deno.land/", vec!["http://deno.land/B.ts"]), + ("https://deno.land/", vec!["https://deno.land/mod/A.ts"]), + ( + "https://deno.land:8080/", + vec!["https://deno.land:8080/C.ts"], + ), + ("https://localhost/", vec!["https://localhost/mod/A.ts"]), + ("https://other/", vec!["https://other/A.ts"]), + ], + ); + } + + fn run_partition_by_root_specifiers_test( + input: Vec<&str>, + expected: Vec<(&str, Vec<&str>)>, + ) { + let input = input + .iter() + .map(|s| ModuleSpecifier::parse(s).unwrap()) + .collect::>(); + let output = partition_by_root_specifiers(input.iter()); + // the assertion is much easier to compare when everything is strings + let output = output + .into_iter() + .map(|(s, vec)| { + ( + s.to_string(), + vec.into_iter().map(|s| s.to_string()).collect::>(), + ) + }) + .collect::>(); + let expected = expected + .into_iter() + .map(|(s, vec)| { + ( + s.to_string(), + vec.into_iter().map(|s| s.to_string()).collect::>(), + ) + }) + .collect::>(); + assert_eq!(output, expected); + } + + #[test] + fn should_get_dir_name_root() { + run_test("http://deno.land/x/test", "deno.land/x/test"); + run_test("http://localhost", "localhost"); + run_test("http://localhost/test%20:test", "localhost/test%20_test"); + + fn run_test(specifier: &str, expected: &str) { + assert_eq!( + dir_name_for_root(&ModuleSpecifier::parse(specifier).unwrap()), + PathBuf::from(expected) + ); + } + } + + #[test] + fn test_unique_path() { + let mut paths = HashSet::new(); + assert_eq!( + get_unique_path(PathBuf::from("/test"), &mut paths), + PathBuf::from("/test") + ); + assert_eq!( + get_unique_path(PathBuf::from("/test"), &mut paths), + PathBuf::from("/test_2") + ); + assert_eq!( + get_unique_path(PathBuf::from("/test"), &mut paths), + PathBuf::from("/test_3") + ); + assert_eq!( + get_unique_path(PathBuf::from("/TEST"), &mut paths), + PathBuf::from("/TEST_4") + ); + assert_eq!( + get_unique_path(PathBuf::from("/test.txt"), &mut paths), + PathBuf::from("/test.txt") + ); + assert_eq!( + get_unique_path(PathBuf::from("/test.txt"), &mut paths), + PathBuf::from("/test_2.txt") + ); + assert_eq!( + get_unique_path(PathBuf::from("/TEST.TXT"), &mut paths), + PathBuf::from("/TEST_3.TXT") + ); + } +} diff --git a/cli/tools/vendor/test.rs b/cli/tools/vendor/test.rs new file mode 100644 index 0000000000000..b37e2b3b0b940 --- /dev/null +++ b/cli/tools/vendor/test.rs @@ -0,0 +1,240 @@ +// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. + +use std::cell::RefCell; +use std::collections::HashMap; +use std::collections::HashSet; +use std::path::Path; +use std::path::PathBuf; +use std::sync::Arc; + +use deno_ast::ModuleSpecifier; +use deno_core::anyhow::anyhow; +use deno_core::anyhow::bail; +use deno_core::error::AnyError; +use deno_core::futures; +use deno_core::serde_json; +use deno_graph::source::LoadFuture; +use deno_graph::source::LoadResponse; +use deno_graph::source::Loader; +use deno_graph::ModuleGraph; + +use super::build::VendorEnvironment; + +// Utilities that help `deno vendor` get tested in memory. + +type RemoteFileText = String; +type RemoteFileHeaders = Option>; +type RemoteFileResult = Result<(RemoteFileText, RemoteFileHeaders), String>; + +#[derive(Clone, Default)] +pub struct TestLoader { + files: HashMap, + redirects: HashMap, +} + +impl TestLoader { + pub fn add( + &mut self, + path_or_specifier: impl AsRef, + text: impl AsRef, + ) -> &mut Self { + if path_or_specifier + .as_ref() + .to_lowercase() + .starts_with("http") + { + self.files.insert( + ModuleSpecifier::parse(path_or_specifier.as_ref()).unwrap(), + Ok((text.as_ref().to_string(), None)), + ); + } else { + let path = make_path(path_or_specifier.as_ref()); + let specifier = ModuleSpecifier::from_file_path(path).unwrap(); + self + .files + .insert(specifier, Ok((text.as_ref().to_string(), None))); + } + self + } + + pub fn add_with_headers( + &mut self, + specifier: impl AsRef, + text: impl AsRef, + headers: &[(&str, &str)], + ) -> &mut Self { + let headers = headers + .iter() + .map(|(key, value)| (key.to_string(), value.to_string())) + .collect(); + self.files.insert( + ModuleSpecifier::parse(specifier.as_ref()).unwrap(), + Ok((text.as_ref().to_string(), Some(headers))), + ); + self + } + + pub fn add_redirect( + &mut self, + from: impl AsRef, + to: impl AsRef, + ) -> &mut Self { + self.redirects.insert( + ModuleSpecifier::parse(from.as_ref()).unwrap(), + ModuleSpecifier::parse(to.as_ref()).unwrap(), + ); + self + } +} + +impl Loader for TestLoader { + fn load( + &mut self, + specifier: &ModuleSpecifier, + _is_dynamic: bool, + ) -> LoadFuture { + let specifier = self.redirects.get(specifier).unwrap_or(specifier); + let result = self.files.get(specifier).map(|result| match result { + Ok(result) => Ok(LoadResponse::Module { + specifier: specifier.clone(), + content: Arc::new(result.0.clone()), + maybe_headers: result.1.clone(), + }), + Err(err) => Err(err), + }); + let result = match result { + Some(Ok(result)) => Ok(Some(result)), + Some(Err(err)) => Err(anyhow!("{}", err)), + None if specifier.scheme() == "data" => { + deno_graph::source::load_data_url(specifier) + } + None => Ok(None), + }; + Box::pin(futures::future::ready(result)) + } +} + +#[derive(Default)] +struct TestVendorEnvironment { + directories: RefCell>, + files: RefCell>, +} + +impl VendorEnvironment for TestVendorEnvironment { + fn create_dir_all(&self, dir_path: &Path) -> Result<(), AnyError> { + let mut directories = self.directories.borrow_mut(); + for path in dir_path.ancestors() { + if !directories.insert(path.to_path_buf()) { + break; + } + } + Ok(()) + } + + fn write_file(&self, file_path: &Path, text: &str) -> Result<(), AnyError> { + let parent = file_path.parent().unwrap(); + if !self.directories.borrow().contains(parent) { + bail!("Directory not found: {}", parent.display()); + } + self + .files + .borrow_mut() + .insert(file_path.to_path_buf(), text.to_string()); + Ok(()) + } +} + +pub struct VendorOutput { + pub files: Vec<(String, String)>, + pub import_map: Option, +} + +#[derive(Default)] +pub struct VendorTestBuilder { + entry_points: Vec, + loader: TestLoader, +} + +impl VendorTestBuilder { + pub fn with_default_setup() -> Self { + let mut builder = VendorTestBuilder::default(); + builder.add_entry_point("/mod.ts"); + builder + } + + pub fn add_entry_point(&mut self, entry_point: impl AsRef) -> &mut Self { + let entry_point = make_path(entry_point.as_ref()); + self + .entry_points + .push(ModuleSpecifier::from_file_path(entry_point).unwrap()); + self + } + + pub async fn build(&mut self) -> Result { + let graph = self.build_graph().await; + let output_dir = make_path("/vendor"); + let environment = TestVendorEnvironment::default(); + super::build::build(&graph, &output_dir, &environment)?; + let mut files = environment.files.borrow_mut(); + let import_map = files.remove(&output_dir.join("import_map.json")); + let mut files = files + .iter() + .map(|(path, text)| (path_to_string(path), text.clone())) + .collect::>(); + + files.sort_by(|a, b| a.0.cmp(&b.0)); + + Ok(VendorOutput { + import_map: import_map.map(|text| serde_json::from_str(&text).unwrap()), + files, + }) + } + + pub fn with_loader(&mut self, action: impl Fn(&mut TestLoader)) -> &mut Self { + action(&mut self.loader); + self + } + + async fn build_graph(&mut self) -> ModuleGraph { + let graph = deno_graph::create_graph( + self + .entry_points + .iter() + .map(|s| (s.to_owned(), deno_graph::ModuleKind::Esm)) + .collect(), + false, + None, + &mut self.loader, + None, + None, + None, + None, + ) + .await; + graph.lock().unwrap(); + graph.valid().unwrap(); + graph + } +} + +fn make_path(text: &str) -> PathBuf { + // This should work all in memory. We're waiting on + // https://github.com/servo/rust-url/issues/730 to provide + // a cross platform path here + assert!(text.starts_with('/')); + if cfg!(windows) { + PathBuf::from(format!("C:{}", text.replace("/", "\\"))) + } else { + PathBuf::from(text) + } +} + +fn path_to_string(path: &Path) -> String { + // inverse of the function above + let path = path.to_string_lossy(); + if cfg!(windows) { + path.replace("C:\\", "\\").replace('\\', "/") + } else { + path.to_string() + } +} diff --git a/ext/http/lib.rs b/ext/http/lib.rs index e11d42da1745c..312942303a4e4 100644 --- a/ext/http/lib.rs +++ b/ext/http/lib.rs @@ -39,7 +39,6 @@ use hyper::service::Service; use hyper::Body; use hyper::Request; use hyper::Response; -use percent_encoding::percent_encode; use serde::Deserialize; use serde::Serialize; use std::borrow::Cow; @@ -428,7 +427,7 @@ fn req_url( // httpie uses http+unix://[percent_encoding_of_path]/ which we follow #[cfg(unix)] HttpSocketAddr::UnixSocket(addr) => Cow::Owned( - percent_encode( + percent_encoding::percent_encode( addr .as_pathname() .and_then(|x| x.to_str()) diff --git a/runtime/ops/http.rs b/runtime/ops/http.rs index 53a99bd47d04d..5b8acb881b5a1 100644 --- a/runtime/ops/http.rs +++ b/runtime/ops/http.rs @@ -8,7 +8,6 @@ use deno_core::OpState; use deno_core::ResourceId; use deno_http::http_create_conn_resource; use deno_net::io::TcpStreamResource; -use deno_net::io::UnixStreamResource; use deno_net::ops_tls::TlsStreamResource; pub fn init() -> Extension { @@ -49,7 +48,7 @@ fn op_http_start( #[cfg(unix)] if let Ok(resource_rc) = state .resource_table - .take::(tcp_stream_rid) + .take::(tcp_stream_rid) { super::check_unstable(state, "Deno.serveHttp");