From c931f7a425846364b4a68173b4989e1e9dc76d59 Mon Sep 17 00:00:00 2001 From: andreespirela Date: Wed, 6 Dec 2023 13:37:13 -0500 Subject: [PATCH 1/2] feat: Include eszip extractor, refactor references for payload, fs crate --- Cargo.toml | 2 +- crates/base/Cargo.toml | 4 +- .../test_cases/npm/folder1/folder2/numbers.ts | 4 + crates/base/test_cases/npm/hello.js | 1 + crates/base/test_cases/npm/index.ts | 4 +- crates/cli/src/main.rs | 23 ++- crates/sb_core/util/path.rs | 21 ++- crates/sb_graph/lib.rs | 139 +++++++++++++++++- crates/sb_module_loader/standalone/mod.rs | 22 +-- 9 files changed, 192 insertions(+), 28 deletions(-) create mode 100644 crates/base/test_cases/npm/folder1/folder2/numbers.ts create mode 100644 crates/base/test_cases/npm/hello.js diff --git a/Cargo.toml b/Cargo.toml index ddae45e4a..6477005a8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,7 +27,7 @@ deno_core = { version = "0.222.0" } deno_console = { version = "0.121.0" } deno_crypto = { version = "0.135.0" } deno_fetch = { version = "0.145.0" } -deno_fs = "0.31.0" +deno_fs = { version = "0.31.0", features = ["sync_fs"] } deno_config = "=0.3.1" deno_io = "0.31.0" deno_graph = "=0.55.0" diff --git a/crates/base/Cargo.toml b/crates/base/Cargo.toml index 67bb162c0..ca1bff706 100644 --- a/crates/base/Cargo.toml +++ b/crates/base/Cargo.toml @@ -17,7 +17,7 @@ anyhow = { workspace = true } bytes = { version = "1.2.1" } cityhash = { version = "0.1.1" } deno_ast = { workspace = true } -deno_fs = { workspace = true, features = ["sync_fs"] } +deno_fs.workspace = true deno_io = { workspace = true } deno_core = { workspace = true } deno_console = { workspace = true } @@ -69,7 +69,7 @@ sb_node = { version = "0.1.0", path = "../node" } anyhow = { workspace = true } bytes = { version = "1.2.1" } deno_ast = { workspace = true } -deno_fs = { workspace = true, features = ["sync_fs"] } +deno_fs.workspace = true deno_io = { workspace = true } deno_core = { workspace = true } deno_console = { workspace = true } diff --git a/crates/base/test_cases/npm/folder1/folder2/numbers.ts b/crates/base/test_cases/npm/folder1/folder2/numbers.ts new file mode 100644 index 000000000..5a2956127 --- /dev/null +++ b/crates/base/test_cases/npm/folder1/folder2/numbers.ts @@ -0,0 +1,4 @@ +export const numbers = { + "Uno": 1, + "Dos": 2 +} \ No newline at end of file diff --git a/crates/base/test_cases/npm/hello.js b/crates/base/test_cases/npm/hello.js new file mode 100644 index 000000000..3ce043d67 --- /dev/null +++ b/crates/base/test_cases/npm/hello.js @@ -0,0 +1 @@ +export const hello = ""; \ No newline at end of file diff --git a/crates/base/test_cases/npm/index.ts b/crates/base/test_cases/npm/index.ts index 387ed07f2..5c42b200c 100644 --- a/crates/base/test_cases/npm/index.ts +++ b/crates/base/test_cases/npm/index.ts @@ -1,9 +1,11 @@ import isEven from "npm:is-even"; import { serve } from "https://deno.land/std@0.131.0/http/server.ts" +import { hello } from "./hello.js"; +import { numbers } from "./folder1/folder2/numbers.ts" serve(async (req: Request) => { return new Response( - JSON.stringify({ is_even: isEven(10) }), + JSON.stringify({ is_even: isEven(10), hello, numbers }), { status: 200, headers: { "Content-Type": "application/json" } }, ) }) \ No newline at end of file diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rs index 0c651a854..c7a5950d0 100644 --- a/crates/cli/src/main.rs +++ b/crates/cli/src/main.rs @@ -7,8 +7,8 @@ use clap::builder::FalseyValueParser; use clap::{arg, crate_version, value_parser, ArgAction, Command}; use deno_core::url::Url; use sb_graph::emitter::EmitterFactory; -use sb_graph::generate_binary_eszip; use sb_graph::import_map::load_import_map; +use sb_graph::{extract_from_file, generate_binary_eszip}; use std::fs::File; use std::io::Write; use std::path::PathBuf; @@ -58,7 +58,12 @@ fn cli() -> Command { .arg(arg!(--"output" "Path to output eszip file").default_value("bin.eszip")) .arg(arg!(--"entrypoint" "Path to entrypoint to bundle as an eszip").required(true)) .arg(arg!(--"import-map" "Path to import map file")) - ) + ).subcommand( + Command::new("unbundle") + .about("Unbundles an .eszip file into the specified directory") + .arg(arg!(--"output" "Path to extract the ESZIP content").default_value("./")) + .arg(arg!(--"eszip" "Path of eszip to extract").required(true)) + ) } //async fn exit_with_code(result: Result<(), Error>) { @@ -162,6 +167,20 @@ fn main() -> Result<(), anyhow::Error> { let mut file = File::create(output_path.as_str()).unwrap(); file.write_all(&bin).unwrap(); } + Some(("unbundle", sub_matches)) => { + let output_path = sub_matches.get_one::("output").cloned().unwrap(); + let eszip_path = sub_matches.get_one::("eszip").cloned().unwrap(); + + let output_path = PathBuf::from(output_path.as_str()); + let eszip_path = PathBuf::from(eszip_path.as_str()); + + extract_from_file(eszip_path, output_path.clone()).await; + + println!( + "Eszip extracted successfully inside path {}", + output_path.to_str().unwrap() + ); + } _ => { // unrecognized command } diff --git a/crates/sb_core/util/path.rs b/crates/sb_core/util/path.rs index 8bbb7825e..505f9aeae 100644 --- a/crates/sb_core/util/path.rs +++ b/crates/sb_core/util/path.rs @@ -1,5 +1,5 @@ use deno_ast::ModuleSpecifier; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; /// Gets if the provided character is not supported on all /// kinds of file systems. @@ -41,3 +41,22 @@ pub fn root_url_to_safe_local_dirname(root: &ModuleSpecifier) -> PathBuf { result } + +pub fn find_lowest_path(paths: &Vec) -> Option { + let mut lowest_path: Option<(&str, usize)> = None; + + for path_str in paths { + // Extract the path part from the URL + let path = Path::new(path_str); + + // Count the components + let component_count = path.components().count(); + + // Update the lowest path if this one has fewer components + if lowest_path.is_none() || component_count < lowest_path.unwrap().1 { + lowest_path = Some((path_str, component_count)); + } + } + + lowest_path.map(|(path, _)| path.to_string()) +} diff --git a/crates/sb_graph/lib.rs b/crates/sb_graph/lib.rs index acd3fa0db..19b73476d 100644 --- a/crates/sb_graph/lib.rs +++ b/crates/sb_graph/lib.rs @@ -2,12 +2,16 @@ use crate::emitter::EmitterFactory; use crate::graph_util::{create_eszip_from_graph_raw, create_graph}; use deno_ast::MediaType; use deno_core::error::AnyError; +use deno_core::futures::io::{AllowStdIo, BufReader}; use deno_core::{serde_json, FastString, JsBuffer, ModuleSpecifier}; use deno_fs::{FileSystem, RealFs}; use deno_npm::NpmSystemInfo; use eszip::{EszipV2, ModuleKind}; use sb_fs::{build_vfs, VfsOpts}; -use std::path::PathBuf; +use std::fs; +use std::fs::{create_dir_all, File}; +use std::io::Write; +use std::path::{Path, PathBuf}; use std::sync::Arc; pub mod emitter; @@ -25,6 +29,26 @@ pub enum EszipPayloadKind { Eszip(EszipV2), } +pub async fn payload_to_eszip(eszip_payload_kind: EszipPayloadKind) -> EszipV2 { + match eszip_payload_kind { + EszipPayloadKind::Eszip(data) => data, + _ => { + let bytes = match eszip_payload_kind { + EszipPayloadKind::JsBufferKind(js_buffer) => Vec::from(&*js_buffer), + EszipPayloadKind::VecKind(vec) => vec, + _ => panic!("It should not get here"), + }; + + let bufreader = BufReader::new(AllowStdIo::new(bytes.as_slice())); + let (eszip, loader) = eszip::EszipV2::parse(bufreader).await.unwrap(); + + loader.await.unwrap(); + + eszip + } + } +} + pub async fn generate_binary_eszip( file: PathBuf, emitter_factory: Arc, @@ -95,3 +119,116 @@ pub async fn generate_binary_eszip( eszip } } + +fn extract_file_specifiers(eszip: &EszipV2) -> Vec { + eszip + .specifiers() + .iter() + .filter(|specifier| specifier.starts_with("file:")) + .cloned() + .collect() +} + +pub struct ExtractEszipPayload { + pub data: EszipPayloadKind, + pub folder: PathBuf, +} + +fn create_module_path( + global_specifier: &str, + entry_path: &Path, + output_folder: &Path, +) -> PathBuf { + let cleaned_specifier = global_specifier.replace(entry_path.to_str().unwrap(), ""); + let module_path = PathBuf::from(cleaned_specifier); + + if let Some(parent) = module_path.parent() { + if parent.parent().is_some() { + let output_folder_and_mod_folder = + output_folder.join(parent.strip_prefix("/").unwrap()); + if !output_folder_and_mod_folder.exists() { + create_dir_all(&output_folder_and_mod_folder).unwrap(); + } + } + } + + output_folder.join(module_path.strip_prefix("/").unwrap()) +} + +async fn extract_modules( + eszip: &EszipV2, + specifiers: &[String], + lowest_path: &str, + output_folder: &Path, +) { + let main_path = PathBuf::from(lowest_path); + let entry_path = main_path.parent().unwrap(); + for global_specifier in specifiers { + let module_path = create_module_path(global_specifier, entry_path, output_folder); + let module_content = eszip + .get_module(global_specifier) + .unwrap() + .take_source() + .await + .unwrap(); + + let mut file = File::create(&module_path).unwrap(); + file.write_all(module_content.as_ref()).unwrap(); + } +} + +pub async fn extract_eszip(payload: ExtractEszipPayload) { + let eszip = payload_to_eszip(payload.data).await; + let output_folder = payload.folder; + + if !output_folder.exists() { + create_dir_all(&output_folder).unwrap(); + } + + let file_specifiers = extract_file_specifiers(&eszip); + if let Some(lowest_path) = sb_core::util::path::find_lowest_path(&file_specifiers) { + extract_modules(&eszip, &file_specifiers, &lowest_path, &output_folder).await; + } else { + panic!("Path seems to be invalid"); + } +} + +pub async fn extract_from_file(eszip_file: PathBuf, output_path: PathBuf) { + let eszip_content = fs::read(eszip_file).expect("File does not exist"); + extract_eszip(ExtractEszipPayload { + data: EszipPayloadKind::VecKind(eszip_content), + folder: output_path, + }) + .await; +} + +#[cfg(test)] +mod test { + use crate::{ + extract_eszip, generate_binary_eszip, EmitterFactory, EszipPayloadKind, ExtractEszipPayload, + }; + use std::fs::remove_dir_all; + use std::path::PathBuf; + use std::sync::Arc; + + #[tokio::test] + #[allow(clippy::arc_with_non_send_sync)] + async fn test_module_code_no_eszip() { + let eszip = generate_binary_eszip( + PathBuf::from("../base/test_cases/npm/index.ts"), + Arc::new(EmitterFactory::new()), + None, + None, + ) + .await; + let eszip = eszip.unwrap(); + extract_eszip(ExtractEszipPayload { + data: EszipPayloadKind::Eszip(eszip), + folder: PathBuf::from("../base/test_cases/extracted-npm/"), + }) + .await; + + assert!(PathBuf::from("../base/test_cases/extracted-npm/hello.js").exists()); + remove_dir_all(PathBuf::from("../base/test_cases/extracted-npm/")).unwrap(); + } +} diff --git a/crates/sb_module_loader/standalone/mod.rs b/crates/sb_module_loader/standalone/mod.rs index 279082c70..df9b04a96 100644 --- a/crates/sb_module_loader/standalone/mod.rs +++ b/crates/sb_module_loader/standalone/mod.rs @@ -20,7 +20,7 @@ use sb_core::util::http_util::HttpClient; use sb_fs::file_system::DenoCompileFileSystem; use sb_fs::load_npm_vfs; use sb_graph::graph_resolver::MappedSpecifierResolver; -use sb_graph::{EszipPayloadKind, SOURCE_CODE_ESZIP_KEY, VFS_ESZIP_KEY}; +use sb_graph::{payload_to_eszip, EszipPayloadKind, SOURCE_CODE_ESZIP_KEY, VFS_ESZIP_KEY}; use sb_node::analyze::NodeCodeTranslator; use sb_node::NodeResolver; use sb_npm::package_json::PackageJsonDepsProvider; @@ -198,25 +198,7 @@ pub async fn create_module_loader_for_standalone_from_eszip_kind( maybe_import_map_arc: Option>, maybe_import_map_path: Option, ) -> Result { - use deno_core::futures::io::{AllowStdIo, BufReader}; - - let eszip = match eszip_payload_kind { - EszipPayloadKind::Eszip(data) => data, - _ => { - let bytes = match eszip_payload_kind { - EszipPayloadKind::JsBufferKind(js_buffer) => Vec::from(&*js_buffer), - EszipPayloadKind::VecKind(vec) => vec, - _ => panic!("It should not get here"), - }; - - let bufreader = BufReader::new(AllowStdIo::new(bytes.as_slice())); - let (eszip, loader) = eszip::EszipV2::parse(bufreader).await.unwrap(); - - loader.await.unwrap(); - - eszip - } - }; + let eszip = payload_to_eszip(eszip_payload_kind).await; let mut maybe_import_map: Option = None; From 2fbae0ca1598011ace017e9a0caa38874f8c8fde Mon Sep 17 00:00:00 2001 From: andreespirela Date: Wed, 6 Dec 2023 13:52:18 -0500 Subject: [PATCH 2/2] chore: Refactor printlns to logs --- Cargo.lock | 2 ++ .../base/src/rt_worker/implementation/default_handler.rs | 2 +- crates/base/tests/user_worker_tests.rs | 5 ++++- crates/cpu_timer/src/lib.rs | 2 +- crates/node/Cargo.toml | 1 + crates/node/global.rs | 4 ++-- crates/node/ops/http2.rs | 1 - crates/npm/Cargo.toml | 3 ++- crates/npm/installer.rs | 5 +++-- crates/npm/registry.rs | 3 ++- crates/npm/resolution.rs | 7 ++++--- crates/sb_graph/graph_resolver.rs | 1 - crates/sb_graph/lib.rs | 6 +----- 13 files changed, 23 insertions(+), 19 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 920043b22..7042439f4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3738,6 +3738,7 @@ dependencies = [ "lazy-regex", "libc", "libz-sys", + "log", "md-5", "md4", "num-bigint", @@ -3787,6 +3788,7 @@ dependencies = [ "deno_semver 0.5.1", "flate2", "hex", + "log", "once_cell", "percent-encoding", "ring", diff --git a/crates/base/src/rt_worker/implementation/default_handler.rs b/crates/base/src/rt_worker/implementation/default_handler.rs index b57f66955..d22fa4d8e 100644 --- a/crates/base/src/rt_worker/implementation/default_handler.rs +++ b/crates/base/src/rt_worker/implementation/default_handler.rs @@ -9,7 +9,7 @@ use tokio::sync::oneshot::Receiver; impl WorkerHandler for Worker { fn handle_error(&self, error: Error) -> Result { - println!("{}", error); + log::error!("{}", error); Ok(WorkerEvents::BootFailure(BootFailureEvent { msg: error.to_string(), })) diff --git a/crates/base/tests/user_worker_tests.rs b/crates/base/tests/user_worker_tests.rs index 6cb3e3352..efcbf0207 100644 --- a/crates/base/tests/user_worker_tests.rs +++ b/crates/base/tests/user_worker_tests.rs @@ -71,5 +71,8 @@ async fn test_user_imports_npm() { let body_bytes = hyper::body::to_bytes(res.into_body()).await.unwrap(); - assert_eq!(body_bytes, r#"{"is_even":true}"#); + assert_eq!( + body_bytes, + r#"{"is_even":true,"hello":"","numbers":{"Uno":1,"Dos":2}}"# + ); } diff --git a/crates/cpu_timer/src/lib.rs b/crates/cpu_timer/src/lib.rs index 95a64de12..03bac985d 100644 --- a/crates/cpu_timer/src/lib.rs +++ b/crates/cpu_timer/src/lib.rs @@ -77,7 +77,7 @@ impl CPUTimer { #[cfg(not(target_os = "linux"))] pub fn start(_: u64, _: u64, _: CPUAlarmVal) -> Result { - println!("CPU timer: not enabled (need Linux)"); + log::error!("CPU timer: not enabled (need Linux)"); Ok(Self {}) } } diff --git a/crates/node/Cargo.toml b/crates/node/Cargo.toml index 756631a38..11cf0c294 100644 --- a/crates/node/Cargo.toml +++ b/crates/node/Cargo.toml @@ -70,3 +70,4 @@ winapi = "=0.3.9" # https://github.com/dalek-cryptography/x25519-dalek/pull/89 x25519-dalek = "2.0.0-pre.1" x509-parser = "0.15.0" +log.workspace = true diff --git a/crates/node/global.rs b/crates/node/global.rs index e7a51cb69..e486b2bf7 100644 --- a/crates/node/global.rs +++ b/crates/node/global.rs @@ -248,13 +248,13 @@ fn is_managed_key(scope: &mut v8::HandleScope, key: v8::Local) -> bool fn current_mode(scope: &mut v8::HandleScope) -> Mode { let Some(v8_string) = v8::StackTrace::current_script_name_or_source_url(scope) else { - println!("current_script_name_or_source_url, Using Deno"); + log::debug!("current_script_name_or_source_url, Using SB"); return Mode::Deno; }; let op_state = deno_core::JsRuntime::op_state_from(scope); let op_state = op_state.borrow(); let Some(node_resolver) = op_state.try_borrow::>() else { - println!("Node resolver not available, using Deno"); + log::debug!("Node resolver not available, using SB"); return Mode::Deno; }; let mut buffer = [MaybeUninit::uninit(); 2048]; diff --git a/crates/node/ops/http2.rs b/crates/node/ops/http2.rs index cde9ddf4b..6cc33fce8 100644 --- a/crates/node/ops/http2.rs +++ b/crates/node/ops/http2.rs @@ -493,7 +493,6 @@ pub async fn op_http2_client_get_response_body_chunk( return Ok((Some(data.to_vec()), false)); } DataOrTrailers::Trailers(trailers) => { - println!("{trailers:?}"); if let Some(trailers_tx) = RcRef::map(&resource, |r| &r.trailers_tx) .borrow_mut() .await diff --git a/crates/npm/Cargo.toml b/crates/npm/Cargo.toml index 6c20414e7..681668c09 100644 --- a/crates/npm/Cargo.toml +++ b/crates/npm/Cargo.toml @@ -30,4 +30,5 @@ percent-encoding = "=2.3.0" hex = "0.4" base64.workspace = true bincode = "=1.3.3" -thiserror.workspace = true \ No newline at end of file +thiserror.workspace = true +log.workspace = true \ No newline at end of file diff --git a/crates/npm/installer.rs b/crates/npm/installer.rs index 186e4fa86..b4ab783f8 100644 --- a/crates/npm/installer.rs +++ b/crates/npm/installer.rs @@ -10,6 +10,7 @@ use deno_core::futures::StreamExt; use deno_npm::registry::NpmRegistryApi; use deno_npm::registry::NpmRegistryPackageInfoLoadError; use deno_semver::package::PackageReq; +use log::debug; use sb_core::util::sync::AtomicFlag; use super::CliNpmRegistryApi; @@ -91,7 +92,7 @@ impl PackageJsonDepsInstaller { .resolve_pkg_id_from_pkg_req(req) .is_ok() }) { - println!("All package.json deps resolvable. Skipping top level install."); + debug!("All package.json deps resolvable. Skipping top level install."); return Ok(()); // everything is already resolvable } @@ -104,7 +105,7 @@ impl PackageJsonDepsInstaller { .resolve_package_req_as_pending_with_info(req, &info); if let Err(err) = result { if inner.npm_registry_api.mark_force_reload() { - println!("Failed to resolve package. Retrying. Error: {err:#}"); + debug!("Failed to resolve package. Retrying. Error: {err:#}"); // re-initialize reqs_with_info_futures = inner.reqs_with_info_futures(&package_reqs); } else { diff --git a/crates/npm/registry.rs b/crates/npm/registry.rs index bd1a83dd7..e0ddc6c26 100644 --- a/crates/npm/registry.rs +++ b/crates/npm/registry.rs @@ -21,6 +21,7 @@ use deno_core::url::Url; use deno_npm::registry::NpmPackageInfo; use deno_npm::registry::NpmRegistryApi; use deno_npm::registry::NpmRegistryPackageInfoLoadError; +use log::debug; use once_cell::sync::Lazy; use sb_core::cache::CacheSetting; use sb_core::cache::CACHE_PERM; @@ -288,7 +289,7 @@ impl CliNpmRegistryApiInner { &self, name: &str, ) -> Result, AnyError> { - println!("Downloading load_package_info_from_registry_inner"); + debug!("Downloading load_package_info_from_registry_inner"); if *self.cache.cache_setting() == CacheSetting::Only { return Err(custom_error( "NotCached", diff --git a/crates/npm/resolution.rs b/crates/npm/resolution.rs index a1ba55ddd..683f1477b 100644 --- a/crates/npm/resolution.rs +++ b/crates/npm/resolution.rs @@ -34,6 +34,7 @@ use sb_core::util::sync::TaskQueue; use super::registry::CliNpmRegistryApi; use deno_lockfile::Lockfile; +use log::debug; /// Handles updating and storing npm resolution in memory where the underlying /// snapshot can be updated concurrently. Additionally handles updating the lockfile @@ -292,7 +293,7 @@ async fn add_package_reqs_to_snapshot( .iter() .all(|req| snapshot.package_reqs().contains_key(req)) { - println!("Snapshot already up to date. Skipping pending resolution."); + debug!("Snapshot already up to date. Skipping pending resolution."); snapshot } else { let pending_resolver = get_npm_pending_resolver(api); @@ -303,8 +304,8 @@ async fn add_package_reqs_to_snapshot( match result { Ok(snapshot) => snapshot, Err(NpmResolutionError::Resolution(err)) if api.mark_force_reload() => { - println!("{err:#}"); - println!("npm resolution failed. Trying again..."); + log::error!("{err:#}"); + log::warn!("npm resolution failed. Trying again..."); // try again let snapshot = get_new_snapshot(); diff --git a/crates/sb_graph/graph_resolver.rs b/crates/sb_graph/graph_resolver.rs index e90225bde..d8e115ffa 100644 --- a/crates/sb_graph/graph_resolver.rs +++ b/crates/sb_graph/graph_resolver.rs @@ -312,7 +312,6 @@ impl NpmResolver for CliGraphResolver { Ok(nv) => NpmPackageReqResolution::Ok(nv), Err(err) => { if self.npm_registry_api.mark_force_reload() { - println!("Restarting npm specifier resolution to check for new registry information. Error: {:#}", err); NpmPackageReqResolution::ReloadRegistryInfo(err.into()) } else { NpmPackageReqResolution::Err(err.into()) diff --git a/crates/sb_graph/lib.rs b/crates/sb_graph/lib.rs index 19b73476d..273980727 100644 --- a/crates/sb_graph/lib.rs +++ b/crates/sb_graph/lib.rs @@ -134,11 +134,7 @@ pub struct ExtractEszipPayload { pub folder: PathBuf, } -fn create_module_path( - global_specifier: &str, - entry_path: &Path, - output_folder: &Path, -) -> PathBuf { +fn create_module_path(global_specifier: &str, entry_path: &Path, output_folder: &Path) -> PathBuf { let cleaned_specifier = global_specifier.replace(entry_path.to_str().unwrap(), ""); let module_path = PathBuf::from(cleaned_specifier);