diff --git a/Cargo.lock b/Cargo.lock index 89921be50..eda24d68e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -362,11 +362,11 @@ dependencies = [ "reqwest", "sb_core", "sb_env", - "sb_eszip", + "sb_graph", + "sb_module_loader", "sb_node", "sb_npm", "sb_os", - "sb_worker_context", "sb_workers", "serde", "thiserror", @@ -676,6 +676,7 @@ dependencies = [ "clap 4.3.21", "env_logger 0.10.0", "log", + "sb_graph", "tokio", ] @@ -3743,12 +3744,13 @@ dependencies = [ "deno_fs", "deno_http", "deno_net", + "deno_tls", "deno_web", "deno_websocket", "hyper 0.14.27", "sb_node", - "sb_worker_context", "serde", + "thiserror", "tokio", ] @@ -3762,14 +3764,75 @@ dependencies = [ ] [[package]] -name = "sb_eszip" +name = "sb_fs" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "deno_ast", + "deno_core", + "deno_fs", + "deno_io", + "deno_npm", + "deno_semver 0.5.1", + "eszip", + "import_map", + "log", + "module_fetcher", + "once_cell", + "sb_node", + "sb_npm", + "serde", + "thiserror", + "tokio", +] + +[[package]] +name = "sb_graph" +version = "0.1.0" +dependencies = [ + "anyhow", + "deno_ast", + "deno_core", + "deno_fs", + "deno_npm", + "deno_semver 0.5.1", + "deno_web", + "eszip", + "import_map", + "log", + "module_fetcher", + "once_cell", + "sb_core", + "sb_fs", + "sb_node", + "sb_npm", + "serde", + "tokio", +] + +[[package]] +name = "sb_module_loader" version = "0.1.0" dependencies = [ "anyhow", + "deno_ast", "deno_core", + "deno_fs", + "deno_npm", + "deno_semver 0.5.1", + "deno_tls", "eszip", "import_map", "log", + "module_fetcher", + "monch", + "once_cell", + "sb_core", + "sb_fs", + "sb_graph", + "sb_node", + "sb_npm", "serde", "tokio", ] @@ -3870,23 +3933,7 @@ dependencies = [ "deno_core", "libc", "sb_core", - "sb_worker_context", - "serde", -] - -[[package]] -name = "sb_worker_context" -version = "0.1.0" -dependencies = [ - "anyhow", - "deno_core", - "enum-as-inner 0.6.0", - "event_worker", - "hyper 0.14.27", - "sb_eszip", "serde", - "tokio", - "uuid", ] [[package]] @@ -3897,11 +3944,11 @@ dependencies = [ "bytes", "deno_core", "deno_http", + "enum-as-inner 0.6.0", "event_worker", "hyper 0.14.27", "log", - "sb_eszip", - "sb_worker_context", + "sb_graph", "serde", "tokio", "uuid", diff --git a/Cargo.toml b/Cargo.toml index 3e600f1a7..ab1f56f1b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,15 +4,16 @@ members = [ "./crates/cli", "./crates/module_fetcher", "./crates/sb_workers", - "./crates/sb_worker_context", "./crates/sb_env", "./crates/sb_core", "./crates/flaky_test", "./crates/sb_os", "./crates/cpu_timer", "./crates/event_worker", - "./crates/sb_eszip", - "./crates/npm" + "./crates/npm", + "./crates/sb_graph", + "./crates/sb_module_loader", + "./crates/sb_fs" ] resolver = "2" @@ -58,6 +59,7 @@ fs3 = "0.5.0" tokio-util = "0.7.4" uuid = { version = "1.3.0", features = ["v4"] } rsa = { version = "0.7.0", default-features = false, features = ["std", "pem", "hazmat"] } +monch = "=0.4.3" reqwest = { version = "0.11.20", default-features = false, features = ["rustls-tls", "stream", "gzip", "brotli", "socks", "json"] } ring = "=0.16.20" diff --git a/crates/base/Cargo.toml b/crates/base/Cargo.toml index 563e5e38b..35aaaf803 100644 --- a/crates/base/Cargo.toml +++ b/crates/base/Cargo.toml @@ -8,7 +8,7 @@ edition = "2021" [dependencies] async-trait.workspace = true thiserror = "1.0.40" -monch = "=0.4.3" +monch.workspace = true once_cell.workspace = true deno_semver.workspace = true deno_npm.workspace = true @@ -41,13 +41,13 @@ serde = { version = "1.0.149", features = ["derive"] } tokio = { workspace = true } url = { version = "2.3.1" } event_worker ={ version = "0.1.0", path = "../event_worker" } -sb_worker_context = { version = "0.1.0", path = "../sb_worker_context" } sb_workers = { version = "0.1.0", path = "../sb_workers" } sb_env = { version = "0.1.0", path = "../sb_env" } sb_core = { version = "0.1.0", path = "../sb_core" } sb_os = { version = "0.1.0", path = "../sb_os" } -sb_eszip = { version = "0.1.0", path = "../sb_eszip" } sb_npm = { version = "0.1.0", path = "../npm" } +sb_graph = { version = "0.1.0", path = "../sb_graph" } +sb_module_loader = { version = "0.1.0", path = "../sb_module_loader" } urlencoding = { version = "2.1.2" } uuid = { workspace = true } deno_broadcast_channel.workspace = true @@ -87,11 +87,9 @@ serde = { version = "1.0.149", features = ["derive"] } tokio.workspace = true url = { version = "2.3.1" } event_worker ={ version = "0.1.0", path = "../event_worker" } -sb_worker_context = { version = "0.1.0", path = "../sb_worker_context" } sb_workers = { version = "0.1.0", path = "../sb_workers" } sb_env = { version = "0.1.0", path = "../sb_env" } sb_core = { version = "0.1.0", path = "../sb_core" } sb_os = { version = "0.1.0", path = "../sb_os" } sb_node = { version = "0.1.0", path = "../node" } -deno_broadcast_channel.workspace = true -sb_eszip = { version = "0.1.0", path = "../sb_eszip" } +deno_broadcast_channel.workspace = true \ No newline at end of file diff --git a/crates/base/src/deno_runtime.rs b/crates/base/src/deno_runtime.rs index 087a5d04b..ee246c59f 100644 --- a/crates/base/src/deno_runtime.rs +++ b/crates/base/src/deno_runtime.rs @@ -1,12 +1,9 @@ use crate::utils::units::mib_to_bytes; -use crate::js_worker::module_loader; use anyhow::{anyhow, bail, Error}; use deno_core::error::AnyError; use deno_core::url::Url; -use deno_core::{ - located_script_name, serde_v8, JsRuntime, ModuleCode, ModuleId, ModuleLoader, RuntimeOptions, -}; +use deno_core::{located_script_name, serde_v8, JsRuntime, ModuleCode, ModuleId, RuntimeOptions}; use deno_http::DefaultHttpPropertyExtractor; use deno_tls::rustls; use deno_tls::rustls::RootCertStore; @@ -17,7 +14,6 @@ use log::error; use serde::de::DeserializeOwned; use std::collections::HashMap; use std::path::Path; -use std::rc::Rc; use std::sync::Arc; use std::time::Duration; use std::{fmt, fs}; @@ -25,24 +21,25 @@ use tokio::net::UnixStream; use tokio::sync::mpsc; use urlencoding::decode; -use crate::cert::ValueRootCertStoreProvider; -use crate::js_worker::emitter::EmitterFactory; -use crate::js_worker::standalone::create_module_loader_for_standalone_from_eszip_kind; -use crate::{errors_rt, snapshot}; +use crate::snapshot; use event_worker::events::{EventMetadata, WorkerEventWithMetadata}; use event_worker::js_interceptors::sb_events_js_interceptors; use event_worker::sb_user_event_worker; +use module_fetcher::file_fetcher::CacheSetting; use module_fetcher::util::diagnostic::print_import_map_diagnostics; -use module_loader::DefaultModuleLoader; +use sb_core::cert::ValueRootCertStoreProvider; use sb_core::http_start::sb_core_http; use sb_core::net::sb_core_net; use sb_core::permissions::{sb_core_permissions, Permissions}; use sb_core::runtime::sb_core_runtime; use sb_core::sb_core_main_js; use sb_env::sb_env as sb_env_op; +use sb_graph::emitter::EmitterFactory; +use sb_graph::{generate_binary_eszip, EszipPayloadKind}; +use sb_module_loader::standalone::create_module_loader_for_standalone_from_eszip_kind; +use sb_module_loader::RuntimeProviders; use sb_node::deno_node; -use sb_npm::CliNpmResolver; -use sb_worker_context::essentials::{UserWorkerMsgs, WorkerContextInitOpts, WorkerRuntimeOpts}; +use sb_workers::context::{UserWorkerMsgs, WorkerContextInitOpts, WorkerRuntimeOpts}; use sb_workers::sb_user_workers; fn load_import_map(maybe_path: Option) -> Result, Error> { @@ -90,7 +87,7 @@ impl fmt::Debug for DenoRuntimeError { } fn get_error_class_name(e: &AnyError) -> &'static str { - errors_rt::get_error_class_name(e).unwrap_or("Error") + sb_core::errors_rt::get_error_class_name(e).unwrap_or("Error") } fn set_v8_flags() { @@ -107,12 +104,6 @@ fn set_v8_flags() { ); } -pub struct RuntimeProviders { - pub npm_resolver: Arc, - pub module_loader: Rc, - pub fs: Arc, -} - pub struct DenoRuntime { pub js_runtime: JsRuntime, pub env_vars: HashMap, // TODO: does this need to be pub? @@ -122,6 +113,7 @@ pub struct DenoRuntime { impl DenoRuntime { #[allow(clippy::unnecessary_literal_unwrap)] + #[allow(clippy::arc_with_non_send_sync)] pub async fn new(opts: WorkerContextInitOpts) -> Result { let WorkerContextInitOpts { service_path, @@ -135,11 +127,6 @@ impl DenoRuntime { maybe_module_code, } = opts; - // check if the service_path exists - if maybe_entrypoint.is_none() && !service_path.exists() { - bail!("service does not exist {:?}", &service_path) - } - set_v8_flags(); let user_agent = "supabase-edge-runtime".to_string(); @@ -152,6 +139,43 @@ impl DenoRuntime { main_module_url = Url::parse(&maybe_entrypoint.unwrap())?; } + let mut net_access_disabled = false; + let mut allow_remote_modules = true; + if conf.is_user_worker() { + let user_conf = conf.as_user_worker().unwrap(); + net_access_disabled = user_conf.net_access_disabled; + allow_remote_modules = user_conf.allow_remote_modules; + } + + let mut maybe_arc_import_map = None; + + let eszip = if let Some(eszip_payload) = maybe_eszip { + eszip_payload + } else { + let mut emitter_factory = EmitterFactory::new(); + + let cache_strategy = if no_module_cache { + CacheSetting::ReloadAll + } else { + CacheSetting::Use + }; + + emitter_factory.set_file_fetcher_allow_remote(allow_remote_modules); + emitter_factory.set_file_fetcher_cache_strategy(cache_strategy); + + let maybe_import_map = load_import_map(import_map_path.clone())?; + emitter_factory.set_import_map(maybe_import_map); + maybe_arc_import_map = emitter_factory.maybe_import_map.clone(); + + let arc_emitter_factory = Arc::new(emitter_factory); + + let main_module_url_file_path = main_module_url.clone().to_file_path().unwrap(); + let eszip = + generate_binary_eszip(main_module_url_file_path, arc_emitter_factory).await?; + + EszipPayloadKind::Eszip(eszip) + }; + // Create and populate a root cert store based on environment variable. // Reference: https://github.com/denoland/deno/blob/v1.37.0/cli/args/mod.rs#L467 let mut root_cert_store = RootCertStore::empty(); @@ -188,14 +212,6 @@ impl DenoRuntime { } } - let mut net_access_disabled = false; - let mut allow_remote_modules = true; - if conf.is_user_worker() { - let user_conf = conf.as_user_worker().unwrap(); - net_access_disabled = user_conf.net_access_disabled; - allow_remote_modules = user_conf.allow_remote_modules; - } - let root_cert_store_provider: Arc = Arc::new(ValueRootCertStoreProvider::new(root_cert_store.clone())); @@ -207,40 +223,24 @@ impl DenoRuntime { stderr: deno_io::StdioPipe::File(std::fs::File::create("/dev/null")?), }); } - let mut emitter_factory = EmitterFactory::new(); - emitter_factory.init_npm().await; let fs = Arc::new(deno_fs::RealFs); - let import_map = load_import_map(import_map_path)?; - - let rt_providers = if maybe_eszip.is_some() { - create_module_loader_for_standalone_from_eszip_kind(maybe_eszip.unwrap(), import_map) - .await - } else { - let npm_resolver = emitter_factory.npm_resolver().clone(); - - let default_module_loader = DefaultModuleLoader::new( - main_module_url.clone(), - import_map, - emitter_factory, - no_module_cache, - allow_remote_modules, - ) - .await?; - - RuntimeProviders { - npm_resolver, - fs: fs.clone(), - module_loader: Rc::new(default_module_loader), - } - }; + let rt_provider = create_module_loader_for_standalone_from_eszip_kind( + eszip, + maybe_arc_import_map, + import_map_path, + ) + .await?; let RuntimeProviders { npm_resolver, fs: file_system, module_loader, - } = rt_providers; + module_code, + } = rt_provider; + + let mod_code = module_code.or(maybe_module_code); let extensions = vec![ sb_core_permissions::init_ops(net_access_disabled), @@ -347,7 +347,7 @@ impl DenoRuntime { } let main_module_id = js_runtime - .load_main_module(&main_module_url, maybe_module_code) + .load_main_module(&main_module_url, mod_code) .await?; Ok(Self { @@ -426,20 +426,73 @@ impl DenoRuntime { #[cfg(test)] mod test { use crate::deno_runtime::DenoRuntime; - use crate::js_worker::emitter::EmitterFactory; - use crate::standalone::binary::generate_binary_eszip; use deno_core::ModuleCode; - use sb_eszip::module_loader::EszipPayloadKind; - use sb_worker_context::essentials::{ + use sb_graph::emitter::EmitterFactory; + use sb_graph::{generate_binary_eszip, EszipPayloadKind}; + use sb_workers::context::{ MainWorkerRuntimeOpts, UserWorkerMsgs, UserWorkerRuntimeOpts, WorkerContextInitOpts, WorkerRuntimeOpts, }; use std::collections::HashMap; + use std::fs; + use std::fs::File; + use std::io::Write; use std::path::PathBuf; use std::sync::Arc; use tokio::net::UnixStream; use tokio::sync::mpsc; + #[tokio::test] + #[allow(clippy::arc_with_non_send_sync)] + async fn test_eszip_with_source_file() { + let (worker_pool_tx, _) = mpsc::unbounded_channel::(); + let mut file = File::create("./test_cases/eszip-source-test.ts").unwrap(); + file.write_all(b"import isEven from \"npm:is-even\"; globalThis.isTenEven = isEven(9);") + .unwrap(); + let path_buf = PathBuf::from("./test_cases/eszip-source-test.ts"); + let emitter_factory = Arc::new(EmitterFactory::new()); + let bin_eszip = generate_binary_eszip(path_buf, emitter_factory.clone()) + .await + .unwrap(); + fs::remove_file("./test_cases/eszip-source-test.ts").unwrap(); + + let eszip_code = bin_eszip.into_bytes(); + + let runtime = DenoRuntime::new(WorkerContextInitOpts { + service_path: PathBuf::from("./test_cases/"), + no_module_cache: false, + import_map_path: None, + env_vars: Default::default(), + events_rx: None, + maybe_eszip: Some(EszipPayloadKind::VecKind(eszip_code)), + maybe_entrypoint: None, + maybe_module_code: None, + conf: { WorkerRuntimeOpts::MainWorker(MainWorkerRuntimeOpts { worker_pool_tx }) }, + }) + .await; + + let mut rt = runtime.unwrap(); + + let main_mod_ev = rt.js_runtime.mod_evaluate(rt.main_module_id); + let _ = rt.js_runtime.run_event_loop(false).await; + + let read_is_even_global = rt + .js_runtime + .execute_script( + "", + ModuleCode::from( + r#" + globalThis.isTenEven; + "# + .to_string(), + ), + ) + .unwrap(); + let read_is_even = rt.to_value::(&read_is_even_global); + assert_eq!(read_is_even.unwrap().to_string(), "false"); + std::mem::drop(main_mod_ev); + } + #[tokio::test] #[allow(clippy::arc_with_non_send_sync)] async fn test_create_eszip_from_graph() { diff --git a/crates/base/src/js_worker/mod.rs b/crates/base/src/js_worker/mod.rs deleted file mode 100644 index 35507de7e..000000000 --- a/crates/base/src/js_worker/mod.rs +++ /dev/null @@ -1,4 +0,0 @@ -pub mod emitter; -pub mod module_loader; -pub mod node_module_loader; -pub mod standalone; diff --git a/crates/base/src/js_worker/module_loader.rs b/crates/base/src/js_worker/module_loader.rs deleted file mode 100644 index a231dff18..000000000 --- a/crates/base/src/js_worker/module_loader.rs +++ /dev/null @@ -1,292 +0,0 @@ -use crate::js_worker::emitter::EmitterFactory; -use crate::js_worker::node_module_loader::ModuleCodeSource; -use crate::utils::graph_util::{create_graph, create_graph_from_specifiers}; -use anyhow::{anyhow, Context, Error}; -use deno_ast::MediaType; -use deno_core::error::{custom_error, AnyError}; -use deno_core::futures::Future; -use deno_core::futures::FutureExt; -use deno_core::ModuleSource; -use deno_core::ModuleSourceFuture; -use deno_core::ModuleSpecifier; -use deno_core::ModuleType; -use deno_core::ResolutionKind; -use deno_core::{ModuleCode, ModuleLoader}; -use eszip::deno_graph; -use eszip::deno_graph::source::Resolver; -use eszip::deno_graph::{EsmModule, JsonModule, Module, ModuleGraph, Resolution}; -use import_map::ImportMap; -use module_fetcher::file_fetcher::CacheSetting; -use module_fetcher::http_util::HttpClient; -use module_fetcher::node; -use module_fetcher::util::text_encoding::code_without_source_map; -use std::pin::Pin; -use std::sync::Arc; - -pub fn make_http_client() -> Result { - let root_cert_store = None; - let unsafely_ignore_certificate_errors = None; - Ok(HttpClient::new( - root_cert_store, - unsafely_ignore_certificate_errors, - )) -} - -struct PreparedModuleLoader { - graph: ModuleGraph, - emitter: Arc, -} - -impl PreparedModuleLoader { - pub fn load_prepared_module( - &self, - specifier: &ModuleSpecifier, - maybe_referrer: Option<&ModuleSpecifier>, - ) -> Result { - if specifier.scheme() == "node" { - unreachable!(); // Node built-in modules should be handled internally. - } - - match self.graph.get(specifier) { - Some(deno_graph::Module::Json(JsonModule { - source, - media_type, - specifier, - .. - })) => Ok(ModuleCodeSource { - code: source.clone().into(), - found_url: specifier.clone(), - media_type: *media_type, - }), - Some(deno_graph::Module::Esm(EsmModule { - source, - media_type, - specifier, - .. - })) => { - let code: ModuleCode = match media_type { - MediaType::JavaScript - | MediaType::Unknown - | MediaType::Cjs - | MediaType::Mjs - | MediaType::Json => source.clone().into(), - MediaType::Dts | MediaType::Dcts | MediaType::Dmts => Default::default(), - MediaType::TypeScript - | MediaType::Mts - | MediaType::Cts - | MediaType::Jsx - | MediaType::Tsx => { - // get emit text - self.emitter.emitter().unwrap().emit_parsed_source( - specifier, - *media_type, - source, - )? - } - MediaType::TsBuildInfo | MediaType::Wasm | MediaType::SourceMap => { - panic!("Unexpected media type {media_type} for {specifier}") - } - }; - - Ok(ModuleCodeSource { - code, - found_url: specifier.clone(), - media_type: *media_type, - }) - } - _ => { - let mut msg = format!("Loading unprepared module: {specifier}"); - if let Some(referrer) = maybe_referrer { - msg = format!("{}, imported from: {}", msg, referrer.as_str()); - } - Err(anyhow!(msg)) - } - } - } - - pub async fn prepare_module_load( - &self, - roots: Vec, - is_dynamic: bool, - ) -> Result<(), AnyError> { - create_graph_from_specifiers(roots, is_dynamic, self.emitter.clone()).await?; - - // If there is a lockfile... - if let Some(lockfile) = self.emitter.get_lock_file() { - let lockfile = lockfile.lock(); - // update it with anything new - lockfile.write().context("Failed writing lockfile.")?; - } - - Ok(()) - } -} - -pub struct DefaultModuleLoader { - graph: ModuleGraph, - prepared_module_loader: Arc, - emitter: Arc, -} - -impl DefaultModuleLoader { - #[allow(clippy::arc_with_non_send_sync)] - pub async fn new( - main_module: ModuleSpecifier, - maybe_import_map: Option, - mut emitter: EmitterFactory, - no_cache: bool, - allow_remote: bool, - ) -> Result { - let cache_setting = if no_cache { - CacheSetting::ReloadAll - } else { - CacheSetting::Use - }; - - emitter.set_file_fetcher_cache_strategy(cache_setting); - emitter.set_file_fetcher_allow_remote(allow_remote); - emitter.set_import_map(maybe_import_map); - - let emitter = Arc::new(emitter); - let graph = create_graph(main_module.to_file_path().unwrap(), emitter.clone()).await; - - Ok(Self { - graph: graph.clone(), - prepared_module_loader: Arc::new(PreparedModuleLoader { - graph, - emitter: emitter.clone(), - }), - emitter, - }) - } - - fn load_sync( - &self, - specifier: &ModuleSpecifier, - maybe_referrer: Option<&ModuleSpecifier>, - _is_dynamic: bool, - ) -> Result { - let code_source = if let Some(result) = self - .emitter - .npm_module_loader() - .load_sync_if_in_npm_package(specifier, maybe_referrer, &*sb_node::allow_all()) - { - result? - } else { - self.prepared_module_loader - .load_prepared_module(specifier, maybe_referrer)? - }; - - let code = code_without_source_map(code_source.code); - - Ok(ModuleSource::new_with_redirect( - match code_source.media_type { - MediaType::Json => ModuleType::Json, - _ => ModuleType::JavaScript, - }, - code, - specifier, - &code_source.found_url, - )) - } -} - -impl ModuleLoader for DefaultModuleLoader { - fn resolve( - &self, - specifier: &str, - referrer: &str, - _kind: ResolutionKind, - ) -> Result { - let cwd = std::env::current_dir().context("Unable to get CWD")?; - let referrer_result = deno_core::resolve_url_or_path(referrer, &cwd); - let permissions = sb_node::allow_all(); - let npm_module_loader = self.emitter.npm_module_loader(); - - if let Ok(referrer) = referrer_result.as_ref() { - if let Some(result) = - npm_module_loader.resolve_if_in_npm_package(specifier, referrer, &*permissions) - { - return result; - } - - let graph = self.graph.clone(); - let maybe_resolved = match graph.get(referrer) { - Some(Module::Esm(module)) => { - module.dependencies.get(specifier).map(|d| &d.maybe_code) - } - _ => None, - }; - - match maybe_resolved { - Some(Resolution::Ok(resolved)) => { - let specifier = &resolved.specifier; - - return match graph.get(specifier) { - Some(Module::Npm(module)) => { - npm_module_loader.resolve_nv_ref(&module.nv_reference, &*permissions) - } - Some(Module::Node(module)) => Ok(module.specifier.clone()), - Some(Module::Esm(module)) => Ok(module.specifier.clone()), - Some(Module::Json(module)) => Ok(module.specifier.clone()), - Some(Module::External(module)) => { - Ok(node::resolve_specifier_into_node_modules(&module.specifier)) - } - None => Ok(specifier.clone()), - }; - } - Some(Resolution::Err(err)) => { - return Err(custom_error( - "TypeError", - format!("{}\n", err.to_string_with_range()), - )) - } - Some(Resolution::None) | None => {} - } - } - - self.emitter - .cli_graph_resolver() - .clone() - .resolve(specifier, &referrer_result?) - } - - fn prepare_load( - &self, - specifier: &ModuleSpecifier, - _maybe_referrer: Option, - is_dynamic: bool, - ) -> Pin>>> { - if let Some(result) = self - .emitter - .npm_module_loader() - .maybe_prepare_load(specifier) - { - return Box::pin(deno_core::futures::future::ready(result)); - } - - let specifier = specifier.clone(); - let module_load_preparer = self.prepared_module_loader.clone(); - - async move { - module_load_preparer - .prepare_module_load(vec![specifier], is_dynamic) - .await - } - .boxed_local() - } - - // TODO: implement prepare_load method - fn load( - &self, - module_specifier: &ModuleSpecifier, - _maybe_referrer: Option<&ModuleSpecifier>, - _is_dyn_import: bool, - ) -> Pin> { - Box::pin(deno_core::futures::future::ready(self.load_sync( - module_specifier, - _maybe_referrer, - _is_dyn_import, - ))) - } -} diff --git a/crates/base/src/lib.rs b/crates/base/src/lib.rs index 344c11933..8e80c8915 100644 --- a/crates/base/src/lib.rs +++ b/crates/base/src/lib.rs @@ -1,13 +1,9 @@ extern crate core; -pub mod cert; pub mod commands; pub mod deno_runtime; -pub mod errors_rt; -pub mod js_worker; pub mod macros; pub mod rt_worker; pub mod server; pub mod snapshot; -pub mod standalone; pub mod utils; diff --git a/crates/base/src/rt_worker/utils.rs b/crates/base/src/rt_worker/utils.rs index c2c634883..6360b672a 100644 --- a/crates/base/src/rt_worker/utils.rs +++ b/crates/base/src/rt_worker/utils.rs @@ -1,5 +1,5 @@ use event_worker::events::{EventMetadata, WorkerEventWithMetadata}; -use sb_worker_context::essentials::{UserWorkerMsgs, WorkerRuntimeOpts}; +use sb_workers::context::{UserWorkerMsgs, WorkerRuntimeOpts}; use tokio::sync::mpsc::UnboundedSender; use uuid::Uuid; diff --git a/crates/base/src/rt_worker/worker.rs b/crates/base/src/rt_worker/worker.rs index 3feb5bb66..518c7a267 100644 --- a/crates/base/src/rt_worker/worker.rs +++ b/crates/base/src/rt_worker/worker.rs @@ -8,7 +8,7 @@ use event_worker::events::{ EventMetadata, ShutdownEvent, UncaughtExceptionEvent, WorkerEventWithMetadata, WorkerEvents, }; use log::{debug, error}; -use sb_worker_context::essentials::{UserWorkerMsgs, WorkerContextInitOpts}; +use sb_workers::context::{UserWorkerMsgs, WorkerContextInitOpts}; use std::any::Any; use std::future::Future; use std::pin::Pin; diff --git a/crates/base/src/rt_worker/worker_ctx.rs b/crates/base/src/rt_worker/worker_ctx.rs index f7b0cd851..28cbe6a75 100644 --- a/crates/base/src/rt_worker/worker_ctx.rs +++ b/crates/base/src/rt_worker/worker_ctx.rs @@ -12,8 +12,8 @@ use event_worker::events::{ }; use hyper::{Body, Request, Response}; use log::{debug, error}; -use sb_eszip::module_loader::EszipPayloadKind; -use sb_worker_context::essentials::{ +use sb_graph::EszipPayloadKind; +use sb_workers::context::{ EventWorkerRuntimeOpts, MainWorkerRuntimeOpts, UserWorkerMsgs, WorkerContextInitOpts, WorkerRequestMsg, WorkerRuntimeOpts, }; diff --git a/crates/base/src/rt_worker/worker_pool.rs b/crates/base/src/rt_worker/worker_pool.rs index 2bb5c89ed..dafba0d2e 100644 --- a/crates/base/src/rt_worker/worker_pool.rs +++ b/crates/base/src/rt_worker/worker_pool.rs @@ -4,7 +4,7 @@ use event_worker::events::WorkerEventWithMetadata; use http::{Request, Response}; use hyper::Body; use log::error; -use sb_worker_context::essentials::{ +use sb_workers::context::{ CreateUserWorkerResult, UserWorkerMsgs, UserWorkerProfile, WorkerContextInitOpts, WorkerRuntimeOpts, }; diff --git a/crates/base/src/server.rs b/crates/base/src/server.rs index f2ee05f19..2347fcb03 100644 --- a/crates/base/src/server.rs +++ b/crates/base/src/server.rs @@ -5,7 +5,7 @@ use anyhow::Error; use event_worker::events::WorkerEventWithMetadata; use hyper::{server::conn::Http, service::Service, Body, Request, Response}; use log::{debug, error, info}; -use sb_worker_context::essentials::WorkerRequestMsg; +use sb_workers::context::WorkerRequestMsg; use std::future::Future; use std::net::IpAddr; use std::net::Ipv4Addr; diff --git a/crates/base/src/standalone/binary.rs b/crates/base/src/standalone/binary.rs deleted file mode 100644 index 560a979cf..000000000 --- a/crates/base/src/standalone/binary.rs +++ /dev/null @@ -1,165 +0,0 @@ -use crate::js_worker::emitter::EmitterFactory; -use crate::standalone::virtual_fs::{FileBackedVfs, VfsBuilder, VfsRoot, VirtualDirectory}; -use crate::standalone::VFS_ESZIP_KEY; -use crate::utils::graph_util::{create_eszip_from_graph_raw, create_graph}; -use deno_core::error::AnyError; -use deno_core::serde_json; -use deno_npm::registry::PackageDepNpmSchemeValueParseError; -use deno_npm::NpmSystemInfo; -use deno_semver::package::PackageReq; -use deno_semver::VersionReqSpecifierParseError; -use eszip::EszipV2; -use module_fetcher::args::package_json::{PackageJsonDepValueParseError, PackageJsonDeps}; -use serde::Deserialize; -use serde::Serialize; -use std::collections::BTreeMap; -use std::path::PathBuf; -use std::sync::Arc; - -#[derive(Serialize, Deserialize)] -enum SerializablePackageJsonDepValueParseError { - SchemeValue(String), - Specifier(String), - Unsupported { scheme: String }, -} - -impl SerializablePackageJsonDepValueParseError { - pub fn from_err(err: PackageJsonDepValueParseError) -> Self { - match err { - PackageJsonDepValueParseError::SchemeValue(err) => Self::SchemeValue(err.value), - PackageJsonDepValueParseError::Specifier(err) => { - Self::Specifier(err.source.to_string()) - } - PackageJsonDepValueParseError::Unsupported { scheme } => Self::Unsupported { scheme }, - } - } - - pub fn into_err(self) -> PackageJsonDepValueParseError { - match self { - SerializablePackageJsonDepValueParseError::SchemeValue(value) => { - PackageJsonDepValueParseError::SchemeValue(PackageDepNpmSchemeValueParseError { - value, - }) - } - SerializablePackageJsonDepValueParseError::Specifier(source) => { - PackageJsonDepValueParseError::Specifier(VersionReqSpecifierParseError { - source: monch::ParseErrorFailureError::new(source), - }) - } - SerializablePackageJsonDepValueParseError::Unsupported { scheme } => { - PackageJsonDepValueParseError::Unsupported { scheme } - } - } - } -} - -#[derive(Serialize, Deserialize)] -pub struct SerializablePackageJsonDeps( - BTreeMap>, -); - -impl SerializablePackageJsonDeps { - pub fn from_deps(deps: PackageJsonDeps) -> Self { - Self( - deps.into_iter() - .map(|(name, req)| { - let res = req.map_err(SerializablePackageJsonDepValueParseError::from_err); - (name, res) - }) - .collect(), - ) - } - - pub fn into_deps(self) -> PackageJsonDeps { - self.0 - .into_iter() - .map(|(name, res)| (name, res.map_err(|err| err.into_err()))) - .collect() - } -} - -#[derive(Deserialize, Serialize)] -pub struct Metadata { - // pub argv: Vec, - // pub unstable: bool, - // pub seed: Option, - // pub location: Option, - // pub v8_flags: Vec, - pub ca_stores: Option>, - pub ca_data: Option>, - pub unsafely_ignore_certificate_errors: Option>, - // pub entrypoint: ModuleSpecifier, - // /// Whether this uses a node_modules directory (true) or the global cache (false). - // pub node_modules_dir: bool, - pub package_json_deps: Option, -} - -pub fn load_npm_vfs(root_dir_path: PathBuf, vfs_data: &[u8]) -> Result { - let mut dir: VirtualDirectory = serde_json::from_slice(vfs_data)?; - - // align the name of the directory with the root dir - dir.name = root_dir_path - .file_name() - .unwrap() - .to_string_lossy() - .to_string(); - - let fs_root = VfsRoot { - dir, - root_path: root_dir_path, - }; - Ok(FileBackedVfs::new(fs_root)) -} - -pub fn build_vfs(emitter_factory: Arc) -> Result { - let npm_resolver = emitter_factory.npm_resolver(); - if let Some(node_modules_path) = npm_resolver.node_modules_path() { - let mut builder = VfsBuilder::new(node_modules_path.clone())?; - builder.add_dir_recursive(&node_modules_path)?; - Ok(builder) - } else { - // DO NOT include the user's registry url as it may contain credentials, - // but also don't make this dependent on the registry url - let registry_url = emitter_factory.npm_api().base_url(); - let root_path = emitter_factory.npm_cache().registry_folder(registry_url); - let mut builder = VfsBuilder::new(root_path)?; - for package in emitter_factory - .npm_resolution() - .all_system_packages(&NpmSystemInfo::default()) - { - let folder = emitter_factory - .npm_resolver() - .resolve_pkg_folder_from_pkg_id(&package.id)?; - builder.add_dir_recursive(&folder)?; - } - // overwrite the root directory's name to obscure the user's registry url - builder.set_root_dir_name("node_modules".to_string()); - Ok(builder) - } -} - -pub async fn generate_binary_eszip( - file: PathBuf, - emitter_factory: Arc, -) -> Result { - let graph = create_graph(file, emitter_factory.clone()).await; - let mut eszip = create_eszip_from_graph_raw(graph, Some(emitter_factory.clone())).await; - - let npm_res = emitter_factory.npm_resolution(); - - let (npm_vfs, _npm_files) = if npm_res.has_packages() { - let (root_dir, files) = build_vfs(emitter_factory.clone())?.into_dir_and_files(); - let snapshot = npm_res.serialized_valid_snapshot_for_system(&NpmSystemInfo::default()); - eszip.add_npm_snapshot(snapshot); - (Some(root_dir), files) - } else { - (None, Vec::new()) - }; - - let npm_vfs = serde_json::to_string(&npm_vfs)?.as_bytes().to_vec(); - let boxed_slice = npm_vfs.into_boxed_slice(); - - eszip.add_opaque_data(String::from(VFS_ESZIP_KEY), Arc::from(boxed_slice)); - - Ok(eszip) -} diff --git a/crates/base/src/standalone/mod.rs b/crates/base/src/standalone/mod.rs deleted file mode 100644 index 0cedf75ff..000000000 --- a/crates/base/src/standalone/mod.rs +++ /dev/null @@ -1,348 +0,0 @@ -// Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. - -use deno_ast::MediaType; -use deno_core::anyhow::Context; -use deno_core::error::generic_error; -use deno_core::error::type_error; -use deno_core::error::AnyError; -use deno_core::futures::FutureExt; -use deno_core::ModuleLoader; -use deno_core::ModuleSpecifier; -use deno_core::ModuleType; -use deno_core::ResolutionKind; -use deno_fs; -use deno_npm::NpmSystemInfo; -use deno_semver::npm::NpmPackageReqReference; -use deno_tls::rustls::RootCertStore; -use deno_tls::RootCertStoreProvider; -use import_map::ImportMap; -use module_fetcher::args::package_json::PackageJsonDepsProvider; -use module_fetcher::args::CacheSetting; -use module_fetcher::cache::{Caches, DenoDirProvider, NodeAnalysisCache}; -use module_fetcher::file_fetcher::get_source_from_data_url; -use module_fetcher::http_util::HttpClient; -use module_fetcher::node::CliCjsCodeAnalyzer; -use sb_node::analyze::NodeCodeTranslator; -use std::pin::Pin; -use std::rc::Rc; -use std::sync::Arc; - -pub mod binary; -pub mod file_system; -pub mod virtual_fs; - -use crate::cert::{get_root_cert_store, CaData}; -use crate::deno_runtime::RuntimeProviders; -use crate::js_worker::node_module_loader::{CjsResolutionStore, NpmModuleLoader}; -use crate::utils::graph_resolver::MappedSpecifierResolver; -use sb_node::NodeResolver; -use sb_npm::{ - create_npm_fs_resolver, CliNpmRegistryApi, CliNpmResolver, NpmCache, NpmCacheDir, NpmResolution, -}; - -use self::binary::load_npm_vfs; -use self::binary::Metadata; -use self::file_system::DenoCompileFileSystem; - -pub const VFS_ESZIP_KEY: &str = "---SUPABASE-VFS-DATA-ESZIP---"; - -pub struct SharedModuleLoaderState { - eszip: eszip::EszipV2, - mapped_specifier_resolver: MappedSpecifierResolver, - npm_module_loader: Arc, -} - -#[derive(Clone)] -pub struct EmbeddedModuleLoader { - shared: Arc, -} - -impl ModuleLoader for EmbeddedModuleLoader { - fn resolve( - &self, - specifier: &str, - referrer: &str, - kind: ResolutionKind, - ) -> Result { - let referrer = if referrer == "." { - if kind != ResolutionKind::MainModule { - return Err(generic_error(format!( - "Expected to resolve main module, got {:?} instead.", - kind - ))); - } - let current_dir = std::env::current_dir().unwrap(); - deno_core::resolve_path(".", ¤t_dir)? - } else { - ModuleSpecifier::parse(referrer) - .map_err(|err| type_error(format!("Referrer uses invalid specifier: {}", err)))? - }; - - let permissions = sb_node::allow_all(); - if let Some(result) = self.shared.npm_module_loader.resolve_if_in_npm_package( - specifier, - &referrer, - &*permissions, - ) { - return result; - } - - let maybe_mapped = self - .shared - .mapped_specifier_resolver - .resolve(specifier, &referrer)? - .into_specifier(); - - // npm specifier - let specifier_text = maybe_mapped - .as_ref() - .map(|r| r.as_str()) - .unwrap_or(specifier); - if let Ok(reference) = NpmPackageReqReference::from_str(specifier_text) { - return self - .shared - .npm_module_loader - .resolve_req_reference(&reference, &*permissions); - } - - match maybe_mapped { - Some(resolved) => Ok(resolved), - None => { - deno_core::resolve_import(specifier, referrer.as_str()).map_err(|err| err.into()) - } - } - } - - fn load( - &self, - original_specifier: &ModuleSpecifier, - maybe_referrer: Option<&ModuleSpecifier>, - _is_dynamic: bool, - ) -> Pin> { - let is_data_uri = get_source_from_data_url(original_specifier).ok(); - let permissions = sb_node::allow_all(); - if let Some((source, _)) = is_data_uri { - return Box::pin(deno_core::futures::future::ready(Ok( - deno_core::ModuleSource::new( - deno_core::ModuleType::JavaScript, - source.into(), - original_specifier, - ), - ))); - } - - if let Some(result) = self.shared.npm_module_loader.load_sync_if_in_npm_package( - original_specifier, - maybe_referrer, - &*permissions, - ) { - return match result { - Ok(code_source) => Box::pin(deno_core::futures::future::ready(Ok( - deno_core::ModuleSource::new_with_redirect( - match code_source.media_type { - MediaType::Json => ModuleType::Json, - _ => ModuleType::JavaScript, - }, - code_source.code, - original_specifier, - &code_source.found_url, - ), - ))), - Err(err) => Box::pin(deno_core::futures::future::ready(Err(err))), - }; - } - - let Some(module) = self.shared.eszip.get_module(original_specifier.as_str()) else { - return Box::pin(deno_core::futures::future::ready(Err(type_error(format!( - "Module not found: {}", - original_specifier - ))))); - }; - let original_specifier = original_specifier.clone(); - let found_specifier = - ModuleSpecifier::parse(&module.specifier).expect("invalid url in eszip"); - - async move { - let code = module - .source() - .await - .ok_or_else(|| type_error(format!("Module not found: {}", original_specifier)))?; - let code = - arc_u8_to_arc_str(code).map_err(|_| type_error("Module source is not utf-8"))?; - Ok(deno_core::ModuleSource::new_with_redirect( - match module.kind { - eszip::ModuleKind::JavaScript => ModuleType::JavaScript, - eszip::ModuleKind::Json => ModuleType::Json, - eszip::ModuleKind::Jsonc => { - return Err(type_error("jsonc modules not supported")) - } - eszip::ModuleKind::OpaqueData => { - unreachable!(); - } - }, - code.into(), - &original_specifier, - &found_specifier, - )) - } - .boxed_local() - } -} - -fn arc_u8_to_arc_str(arc_u8: Arc<[u8]>) -> Result, std::str::Utf8Error> { - // Check that the string is valid UTF-8. - std::str::from_utf8(&arc_u8)?; - // SAFETY: the string is valid UTF-8, and the layout Arc<[u8]> is the same as - // Arc. This is proven by the From> impl for Arc<[u8]> from the - // standard library. - Ok(unsafe { std::mem::transmute(arc_u8) }) -} - -pub struct StandaloneModuleLoaderFactory { - shared: Arc, -} - -struct StandaloneRootCertStoreProvider { - ca_stores: Option>, - ca_data: Option, - cell: once_cell::sync::OnceCell, -} - -impl RootCertStoreProvider for StandaloneRootCertStoreProvider { - fn get_or_try_init(&self) -> Result<&RootCertStore, AnyError> { - self.cell.get_or_try_init(|| { - get_root_cert_store(None, self.ca_stores.clone(), self.ca_data.clone()) - .map_err(|err| err.into()) - }) - } -} - -pub async fn create_module_loader_for_eszip( - mut eszip: eszip::EszipV2, - metadata: Metadata, - maybe_import_map: Option, -) -> Result { - // let main_module = &metadata.entrypoint; - let current_exe_path = std::env::current_exe().unwrap(); - let current_exe_name = current_exe_path.file_name().unwrap().to_string_lossy(); - let deno_dir_provider = Arc::new(DenoDirProvider::new(None)); - let root_cert_store_provider = Arc::new(StandaloneRootCertStoreProvider { - ca_stores: metadata.ca_stores, - ca_data: metadata.ca_data.map(CaData::Bytes), - cell: Default::default(), - }); - let http_client = Arc::new(HttpClient::new( - Some(root_cert_store_provider.clone()), - metadata.unsafely_ignore_certificate_errors.clone(), - )); - - // use a dummy npm registry url - let npm_registry_url = ModuleSpecifier::parse("https://localhost/").unwrap(); - let root_path = std::env::temp_dir() - .join(format!("sb-compile-{}", current_exe_name)) - .join("node_modules"); - let npm_cache_dir = NpmCacheDir::new(root_path.clone()); - let (fs, snapshot) = if let Some(snapshot) = eszip.take_npm_snapshot() { - // TODO: Support node_modules - let vfs_root_dir_path = npm_cache_dir.registry_folder(&npm_registry_url); - - let vfs_data: Vec = eszip - .get_module(VFS_ESZIP_KEY) - .unwrap() - .take_source() - .await - .unwrap() - .to_vec(); - - let vfs = load_npm_vfs(vfs_root_dir_path, &vfs_data).context("Failed to load npm vfs.")?; - - ( - Arc::new(DenoCompileFileSystem::new(vfs)) as Arc, - Some(snapshot), - ) - } else { - ( - Arc::new(deno_fs::RealFs) as Arc, - None, - ) - }; - - let npm_cache = Arc::new(NpmCache::new( - npm_cache_dir, - CacheSetting::Only, - fs.clone(), - http_client.clone(), - )); - - let npm_api = Arc::new(CliNpmRegistryApi::new( - npm_registry_url.clone(), - npm_cache.clone(), - http_client.clone(), - )); - - let npm_resolution = Arc::new(NpmResolution::from_serialized( - npm_api.clone(), - snapshot, - None, - )); - - let npm_fs_resolver = create_npm_fs_resolver( - fs.clone(), - npm_cache, - npm_registry_url, - npm_resolution.clone(), - None, - NpmSystemInfo::default(), - ); - - let npm_resolver = Arc::new(CliNpmResolver::new( - fs.clone(), - npm_resolution.clone(), - npm_fs_resolver, - None, - )); - - let node_resolver = Arc::new(NodeResolver::new(fs.clone(), npm_resolver.clone())); - let cjs_resolutions = Arc::new(CjsResolutionStore::default()); - let cache_db = Caches::new(deno_dir_provider.clone()); - let node_analysis_cache = NodeAnalysisCache::new(cache_db.node_analysis_db()); - let cjs_esm_code_analyzer = CliCjsCodeAnalyzer::new(node_analysis_cache, fs.clone()); - let node_code_translator = Arc::new(NodeCodeTranslator::new( - cjs_esm_code_analyzer, - fs.clone(), - node_resolver.clone(), - npm_resolver.clone(), - )); - let package_json_deps_provider = Arc::new(PackageJsonDepsProvider::new( - metadata - .package_json_deps - .map(|serialized| serialized.into_deps()), - )); - let maybe_import_map = maybe_import_map - .map(|import_map| Some(Arc::new(import_map))) - .unwrap_or_else(|| None); - - let module_loader_factory = StandaloneModuleLoaderFactory { - shared: Arc::new(SharedModuleLoaderState { - eszip, - mapped_specifier_resolver: MappedSpecifierResolver::new( - maybe_import_map, - package_json_deps_provider.clone(), - ), - npm_module_loader: Arc::new(NpmModuleLoader::new( - cjs_resolutions, - node_code_translator, - fs.clone(), - node_resolver.clone(), - )), - }), - }; - - Ok(RuntimeProviders { - module_loader: Rc::new(EmbeddedModuleLoader { - shared: module_loader_factory.shared.clone(), - }), - npm_resolver, - fs, - }) -} diff --git a/crates/base/src/utils.rs b/crates/base/src/utils.rs index 9602fbeee..908cf8813 100644 --- a/crates/base/src/utils.rs +++ b/crates/base/src/utils.rs @@ -1,8 +1,6 @@ use event_worker::events::{EventMetadata, WorkerEventWithMetadata, WorkerEvents}; use tokio::sync::mpsc; -pub mod graph_resolver; -pub mod graph_util; pub mod units; pub fn send_event_if_event_worker_available( diff --git a/crates/base/tests/import_map_tests.rs b/crates/base/tests/import_map_tests.rs index 825969b40..7427b39cc 100644 --- a/crates/base/tests/import_map_tests.rs +++ b/crates/base/tests/import_map_tests.rs @@ -5,7 +5,7 @@ use tokio::sync::oneshot; use urlencoding::encode; use base::rt_worker::worker_ctx::create_worker; -use sb_worker_context::essentials::{ +use sb_workers::context::{ UserWorkerRuntimeOpts, WorkerContextInitOpts, WorkerRequestMsg, WorkerRuntimeOpts, }; diff --git a/crates/base/tests/main_worker_tests.rs b/crates/base/tests/main_worker_tests.rs index cb2ec9ce4..3880a345e 100644 --- a/crates/base/tests/main_worker_tests.rs +++ b/crates/base/tests/main_worker_tests.rs @@ -1,6 +1,6 @@ use base::rt_worker::worker_ctx::{create_user_worker_pool, create_worker}; use hyper::{Body, Request, Response}; -use sb_worker_context::essentials::{ +use sb_workers::context::{ MainWorkerRuntimeOpts, WorkerContextInitOpts, WorkerRequestMsg, WorkerRuntimeOpts, }; use std::collections::HashMap; diff --git a/crates/base/tests/null_body_status_null_body_tests.rs b/crates/base/tests/null_body_status_null_body_tests.rs index 9ee52e081..761fdbb32 100644 --- a/crates/base/tests/null_body_status_null_body_tests.rs +++ b/crates/base/tests/null_body_status_null_body_tests.rs @@ -1,6 +1,6 @@ use base::rt_worker::worker_ctx::create_worker; use hyper::{Body, Request, Response}; -use sb_worker_context::essentials::{ +use sb_workers::context::{ UserWorkerRuntimeOpts, WorkerContextInitOpts, WorkerRequestMsg, WorkerRuntimeOpts, }; use std::collections::HashMap; diff --git a/crates/base/tests/oak_user_worker_tests.rs b/crates/base/tests/oak_user_worker_tests.rs index 9a3cfc1f5..c518b21b6 100644 --- a/crates/base/tests/oak_user_worker_tests.rs +++ b/crates/base/tests/oak_user_worker_tests.rs @@ -1,6 +1,6 @@ use base::rt_worker::worker_ctx::create_worker; use hyper::{Body, Request, Response}; -use sb_worker_context::essentials::{ +use sb_workers::context::{ UserWorkerRuntimeOpts, WorkerContextInitOpts, WorkerRequestMsg, WorkerRuntimeOpts, }; use std::collections::HashMap; diff --git a/crates/base/tests/test_node_server.rs b/crates/base/tests/test_node_server.rs index eedc2ba47..87da6dd05 100644 --- a/crates/base/tests/test_node_server.rs +++ b/crates/base/tests/test_node_server.rs @@ -1,6 +1,6 @@ use base::rt_worker::worker_ctx::create_worker; use hyper::{Body, Request, Response}; -use sb_worker_context::essentials::{ +use sb_workers::context::{ UserWorkerRuntimeOpts, WorkerContextInitOpts, WorkerRequestMsg, WorkerRuntimeOpts, }; use std::collections::HashMap; diff --git a/crates/base/tests/tls_invalid_data_tests.rs b/crates/base/tests/tls_invalid_data_tests.rs index ed3792952..d083c9b8f 100644 --- a/crates/base/tests/tls_invalid_data_tests.rs +++ b/crates/base/tests/tls_invalid_data_tests.rs @@ -1,6 +1,6 @@ use base::rt_worker::worker_ctx::create_worker; use hyper::{Body, Request, Response}; -use sb_worker_context::essentials::{ +use sb_workers::context::{ UserWorkerRuntimeOpts, WorkerContextInitOpts, WorkerRequestMsg, WorkerRuntimeOpts, }; use std::collections::HashMap; diff --git a/crates/base/tests/user_worker_tests.rs b/crates/base/tests/user_worker_tests.rs index 52614dd3e..6cb3e3352 100644 --- a/crates/base/tests/user_worker_tests.rs +++ b/crates/base/tests/user_worker_tests.rs @@ -1,6 +1,6 @@ use base::rt_worker::worker_ctx::create_worker; use hyper::{Body, Request, Response}; -use sb_worker_context::essentials::{ +use sb_workers::context::{ UserWorkerRuntimeOpts, WorkerContextInitOpts, WorkerRequestMsg, WorkerRuntimeOpts, }; use std::collections::HashMap; diff --git a/crates/base/tests/worker_boot_tests.rs b/crates/base/tests/worker_boot_tests.rs index d9639a8fa..e4ad4e7f6 100644 --- a/crates/base/tests/worker_boot_tests.rs +++ b/crates/base/tests/worker_boot_tests.rs @@ -1,9 +1,7 @@ use std::collections::HashMap; use base::rt_worker::worker_ctx::create_worker; -use sb_worker_context::essentials::{ - UserWorkerRuntimeOpts, WorkerContextInitOpts, WorkerRuntimeOpts, -}; +use sb_workers::context::{UserWorkerRuntimeOpts, WorkerContextInitOpts, WorkerRuntimeOpts}; #[tokio::test] async fn test_worker_boot_invalid_imports() { diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index 9dbe75284..9839b7216 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -10,6 +10,7 @@ path = "src/main.rs" [dependencies] anyhow = { workspace = true } base = { path = "../base" } +sb_graph = { path = "../sb_graph" } clap = { version = "4.0.29", features = ["cargo"] } env_logger = "0.10.0" log = { workspace = true } diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rs index 305db9471..0f04d64e2 100644 --- a/crates/cli/src/main.rs +++ b/crates/cli/src/main.rs @@ -2,11 +2,11 @@ mod logger; use anyhow::Error; use base::commands::start_server; -use base::js_worker::emitter::EmitterFactory; use base::server::WorkerEntrypoints; -use base::standalone::binary::generate_binary_eszip; use clap::builder::FalseyValueParser; use clap::{arg, crate_version, value_parser, ArgAction, Command}; +use sb_graph::emitter::EmitterFactory; +use sb_graph::generate_binary_eszip; use std::fs::File; use std::io::Write; use std::path::PathBuf; diff --git a/crates/module_fetcher/Cargo.toml b/crates/module_fetcher/Cargo.toml index 6a80f7eae..f93760d51 100644 --- a/crates/module_fetcher/Cargo.toml +++ b/crates/module_fetcher/Cargo.toml @@ -12,7 +12,7 @@ jsonc-parser = { version="0.21.0", features = ["serde"] } base32 = "=0.4.0" deno_npm.workspace = true deno_semver.workspace = true -monch = "=0.4.3" +monch.workspace = true walkdir = "=2.3.2" fs3.workspace = true regex.workspace = true @@ -20,7 +20,7 @@ once_cell.workspace = true thiserror.workspace = true deno_lockfile.workspace = true indexmap.workspace = true -async-trait = "0.1.73" +async-trait.workspace = true base64 = { version = "=0.13.1" } cache_control = { version = "=0.2.0" } chrono = { version = "=0.4.22", default-features = false, features = ["clock"] } diff --git a/crates/sb_core/Cargo.toml b/crates/sb_core/Cargo.toml index 2ce94166c..08e71d1b8 100644 --- a/crates/sb_core/Cargo.toml +++ b/crates/sb_core/Cargo.toml @@ -24,5 +24,6 @@ deno_http.workspace = true hyper.workspace = true serde.workspace = true bytes.workspace = true -sb_worker_context = { version = "0.1.0", path = "../sb_worker_context" } +deno_tls.workspace = true +thiserror.workspace = true sb_node = { version = "0.1.0", path = "../node" } diff --git a/crates/base/src/cert.rs b/crates/sb_core/cert.rs similarity index 100% rename from crates/base/src/cert.rs rename to crates/sb_core/cert.rs diff --git a/crates/base/src/errors_rt.rs b/crates/sb_core/errors_rt.rs similarity index 100% rename from crates/base/src/errors_rt.rs rename to crates/sb_core/errors_rt.rs diff --git a/crates/sb_core/lib.rs b/crates/sb_core/lib.rs index 96b410964..9af52f4cd 100644 --- a/crates/sb_core/lib.rs +++ b/crates/sb_core/lib.rs @@ -1,3 +1,5 @@ +pub mod cert; +pub mod errors_rt; pub mod http_start; pub mod net; pub mod permissions; diff --git a/crates/sb_eszip/Cargo.toml b/crates/sb_eszip/Cargo.toml deleted file mode 100644 index ba3834f7f..000000000 --- a/crates/sb_eszip/Cargo.toml +++ /dev/null @@ -1,19 +0,0 @@ -[package] -name = "sb_eszip" -version = "0.1.0" -authors = ["Supabase "] -edition = "2021" -resolver = "2" -license = "MIT" - -[lib] -path = "lib.rs" - -[dependencies] -anyhow.workspace = true -deno_core.workspace = true -eszip.workspace = true -import_map = { version = "0.15.0" } -log = { workspace = true } -serde.workspace = true -tokio.workspace = true diff --git a/crates/sb_eszip/lib.rs b/crates/sb_eszip/lib.rs deleted file mode 100644 index 9c80490f3..000000000 --- a/crates/sb_eszip/lib.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod module_loader; diff --git a/crates/sb_eszip/module_loader.rs b/crates/sb_eszip/module_loader.rs deleted file mode 100644 index 8bc97f80b..000000000 --- a/crates/sb_eszip/module_loader.rs +++ /dev/null @@ -1,140 +0,0 @@ -use anyhow::{bail, Error}; -use deno_core::futures::io::{AllowStdIo, BufReader}; -use deno_core::futures::FutureExt; -use deno_core::url::Url; -use deno_core::JsBuffer; -use deno_core::ModuleLoader; -use deno_core::ModuleSource; -use deno_core::ModuleSourceFuture; -use deno_core::ModuleSpecifier; -use deno_core::ResolutionKind; -use import_map::{parse_from_json, ImportMap}; -use log::warn; -use std::path::Path; -use std::pin::Pin; - -pub struct EszipModuleLoader { - eszip: eszip::EszipV2, - maybe_import_map: Option, -} - -#[derive(Debug)] -pub enum EszipPayloadKind { - JsBufferKind(JsBuffer), - VecKind(Vec), -} - -impl EszipModuleLoader { - pub async fn new( - eszip_payload: EszipPayloadKind, - maybe_import_map_url: Option, - ) -> Result { - let bytes = match eszip_payload { - EszipPayloadKind::JsBufferKind(js_buffer) => Vec::from(&*js_buffer), - EszipPayloadKind::VecKind(vec) => vec, - }; - - let bufreader = BufReader::new(AllowStdIo::new(bytes.as_slice())); - let (eszip, loader) = eszip::EszipV2::parse(bufreader).await?; - - loader.await?; - - // load import map - let mut maybe_import_map: Option = None; - if maybe_import_map_url.is_some() { - let import_map_url = Url::parse(&maybe_import_map_url.unwrap())?; - - if let Some(import_map_module) = eszip.get_import_map(import_map_url.as_str()) { - if let Some(source) = import_map_module.source().await { - let source = std::str::from_utf8(&source)?.to_string(); - let result = parse_from_json(&import_map_url, &source)?; - if !result.diagnostics.is_empty() { - warn!( - "Import map diagnostics:\n{}", - result - .diagnostics - .iter() - .map(|d| format!(" - {d}")) - .collect::>() - .join("\n") - ); - } - maybe_import_map = Some(result.import_map); - } - } - } - - Ok(Self { - eszip, - maybe_import_map, - }) - } -} - -impl ModuleLoader for EszipModuleLoader { - fn resolve( - &self, - specifier: &str, - referrer: &str, - _kind: ResolutionKind, - ) -> Result { - if let Some(import_map) = &self.maybe_import_map { - let referrer_relative = Path::new(referrer).is_relative(); - let referrer_url = if referrer_relative { - import_map.base_url().join(referrer) - } else { - Url::parse(referrer) - }; - if referrer_url.is_err() { - return referrer_url.map_err(|err| err.into()); - } - - let referrer_url = referrer_url.unwrap(); - import_map - .resolve(specifier, &referrer_url) - .map_err(|err| err.into()) - } else { - deno_core::resolve_import(specifier, referrer).map_err(|err| err.into()) - } - } - - fn load( - &self, - module_specifier: &ModuleSpecifier, - _maybe_referrer: Option<&ModuleSpecifier>, - _is_dyn_import: bool, - ) -> Pin> { - let maybe_module = self.eszip.get_module(module_specifier.as_str()); - let module_specifier = module_specifier.clone(); - - async move { - if let Some(module) = maybe_module { - if let Some(code) = module.source().await { - let code = std::str::from_utf8(&code)?.to_string(); - let module_type = match module.kind { - eszip::ModuleKind::JavaScript => Some(deno_core::ModuleType::JavaScript), - eszip::ModuleKind::Json => Some(deno_core::ModuleType::Json), - eszip::ModuleKind::Jsonc => None, - eszip::ModuleKind::OpaqueData => None, - }; - if module_type.is_none() { - bail!("invalid module type {}", &module_specifier) - } - let module = ModuleSource::new_with_redirect( - module_type.unwrap(), - code.into(), - &module_specifier, - &Url::parse(&module.specifier)?, - ); - - Ok(module) - } else { - bail!("module source already taken {}", &module_specifier) - } - } else { - bail!("module not found {}", &module_specifier) - } - } - .boxed_local() - } -} diff --git a/crates/sb_fs/Cargo.toml b/crates/sb_fs/Cargo.toml new file mode 100644 index 000000000..71cdf7d48 --- /dev/null +++ b/crates/sb_fs/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "sb_fs" +version = "0.1.0" +authors = ["Supabase "] +edition = "2021" +resolver = "2" +license = "MIT" + +[lib] +path = "lib.rs" + +[dependencies] +module_fetcher = { path = "../module_fetcher" } +sb_node = { version = "0.1.0", path = "../node" } +sb_npm = { version = "0.1.0", path = "../npm" } +deno_semver.workspace = true +anyhow.workspace = true +deno_core.workspace = true +eszip.workspace = true +import_map = { version = "0.15.0" } +log = { workspace = true } +serde.workspace = true +tokio.workspace = true +deno_ast.workspace = true +deno_fs.workspace = true +deno_npm.workspace = true +once_cell.workspace = true +deno_io.workspace = true +thiserror.workspace = true +async-trait.workspace = true \ No newline at end of file diff --git a/crates/base/src/standalone/file_system.rs b/crates/sb_fs/file_system.rs similarity index 100% rename from crates/base/src/standalone/file_system.rs rename to crates/sb_fs/file_system.rs diff --git a/crates/sb_fs/lib.rs b/crates/sb_fs/lib.rs new file mode 100644 index 000000000..40fffca04 --- /dev/null +++ b/crates/sb_fs/lib.rs @@ -0,0 +1,61 @@ +use crate::virtual_fs::{FileBackedVfs, VfsBuilder, VfsRoot, VirtualDirectory}; +use deno_core::error::AnyError; +use deno_core::serde_json; +use deno_npm::NpmSystemInfo; +use sb_npm::{CliNpmRegistryApi, CliNpmResolver, NpmCache, NpmResolution}; +use std::path::PathBuf; +use std::sync::Arc; + +pub mod file_system; +pub mod virtual_fs; + +pub struct VfsOpts { + pub npm_resolver: Arc, + pub npm_registry_api: Arc, + pub npm_cache: Arc, + pub npm_resolution: Arc, +} + +pub fn load_npm_vfs(root_dir_path: PathBuf, vfs_data: &[u8]) -> Result { + let mut dir: VirtualDirectory = serde_json::from_slice(vfs_data)?; + + // align the name of the directory with the root dir + dir.name = root_dir_path + .file_name() + .unwrap() + .to_string_lossy() + .to_string(); + + let fs_root = VfsRoot { + dir, + root_path: root_dir_path, + }; + Ok(FileBackedVfs::new(fs_root)) +} + +pub fn build_vfs(opts: VfsOpts) -> Result { + let npm_resolver = opts.npm_resolver.clone(); + if let Some(node_modules_path) = npm_resolver.node_modules_path() { + let mut builder = VfsBuilder::new(node_modules_path.clone())?; + builder.add_dir_recursive(&node_modules_path)?; + Ok(builder) + } else { + // DO NOT include the user's registry url as it may contain credentials, + // but also don't make this dependent on the registry url + let registry_url = opts.npm_registry_api.base_url(); + let root_path = opts.npm_cache.registry_folder(registry_url); + let mut builder = VfsBuilder::new(root_path)?; + for package in opts + .npm_resolution + .all_system_packages(&NpmSystemInfo::default()) + { + let folder = npm_resolver + .clone() + .resolve_pkg_folder_from_pkg_id(&package.id)?; + builder.add_dir_recursive(&folder)?; + } + // overwrite the root directory's name to obscure the user's registry url + builder.set_root_dir_name("node_modules".to_string()); + Ok(builder) + } +} diff --git a/crates/base/src/standalone/virtual_fs.rs b/crates/sb_fs/virtual_fs.rs similarity index 100% rename from crates/base/src/standalone/virtual_fs.rs rename to crates/sb_fs/virtual_fs.rs diff --git a/crates/sb_graph/Cargo.toml b/crates/sb_graph/Cargo.toml new file mode 100644 index 000000000..9b47e0ad7 --- /dev/null +++ b/crates/sb_graph/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "sb_graph" +version = "0.1.0" +authors = ["Supabase "] +edition = "2021" +resolver = "2" +license = "MIT" + +[lib] +path = "lib.rs" + +[dependencies] +module_fetcher = { path = "../module_fetcher" } +sb_node = { version = "0.1.0", path = "../node" } +sb_npm = { version = "0.1.0", path = "../npm" } +sb_fs = { version = "0.1.0", path = "../sb_fs" } +sb_core = { version = "0.1.0", path = "../sb_core" } +deno_semver.workspace = true +anyhow.workspace = true +deno_core.workspace = true +eszip.workspace = true +import_map = { version = "0.15.0" } +log = { workspace = true } +serde.workspace = true +tokio.workspace = true +deno_ast.workspace = true +deno_fs.workspace = true +deno_npm.workspace = true +once_cell.workspace = true +deno_web.workspace = true \ No newline at end of file diff --git a/crates/base/src/js_worker/emitter.rs b/crates/sb_graph/emitter.rs similarity index 89% rename from crates/base/src/js_worker/emitter.rs rename to crates/sb_graph/emitter.rs index 8f72068c6..c95e7c5d9 100644 --- a/crates/base/src/js_worker/emitter.rs +++ b/crates/sb_graph/emitter.rs @@ -1,6 +1,4 @@ -use crate::js_worker::module_loader::make_http_client; -use crate::js_worker::node_module_loader::{CjsResolutionStore, NpmModuleLoader}; -use crate::utils::graph_resolver::{CliGraphResolver, CliGraphResolverOptions}; +use crate::graph_resolver::{CliGraphResolver, CliGraphResolverOptions}; use deno_ast::EmitOptions; use deno_core::error::AnyError; use deno_core::parking_lot::Mutex; @@ -14,15 +12,13 @@ use module_fetcher::args::package_json::{ }; use module_fetcher::args::CacheSetting; use module_fetcher::cache::{ - Caches, DenoDir, DenoDirProvider, EmitCache, GlobalHttpCache, NodeAnalysisCache, - ParsedSourceCache, RealDenoCacheEnv, + Caches, DenoDir, DenoDirProvider, EmitCache, GlobalHttpCache, ParsedSourceCache, + RealDenoCacheEnv, }; use module_fetcher::emit::Emitter; use module_fetcher::file_fetcher::FileFetcher; use module_fetcher::http_util::HttpClient; -use module_fetcher::node::CliCjsCodeAnalyzer; use module_fetcher::permissions::Permissions; -use sb_node::analyze::NodeCodeTranslator; use sb_node::{NodeResolver, PackageJson}; use sb_npm::{ create_npm_fs_resolver, CliNpmRegistryApi, CliNpmResolver, NpmCache, NpmCacheDir, @@ -80,7 +76,6 @@ pub struct EmitterFactory { deno_dir: DenoDir, pub npm_snapshot: Option, lockfile: Deferred>>>, - cjs_resolutions: Deferred>, package_json_deps_provider: Deferred>, package_json_deps_installer: Deferred>, npm_api: Deferred>, @@ -93,7 +88,7 @@ pub struct EmitterFactory { resolver: Deferred>, file_fetcher_cache_strategy: Option, file_fetcher_allow_remote: bool, - maybe_import_map: Option>, + pub maybe_import_map: Option>, } impl Default for EmitterFactory { @@ -110,7 +105,6 @@ impl EmitterFactory { deno_dir, npm_snapshot: None, lockfile: Default::default(), - cjs_resolutions: Default::default(), package_json_deps_provider: Default::default(), package_json_deps_installer: Default::default(), npm_api: Default::default(), @@ -215,7 +209,12 @@ impl EmitterFactory { } pub fn http_client(&self) -> Arc { - Arc::new(make_http_client().unwrap()) + let root_cert_store = None; + let unsafely_ignore_certificate_errors = None; + + let http_client = HttpClient::new(root_cert_store, unsafely_ignore_certificate_errors); + + Arc::new(http_client) } pub fn real_fs(&self) -> Arc { @@ -282,30 +281,6 @@ impl EmitterFactory { }) } - pub fn cjs_resolution_store(&self) -> &Arc { - self.cjs_resolutions.get_or_init(Default::default) - } - - pub fn npm_module_loader(&self) -> Arc { - let cache_db = Caches::new(self.deno_dir_provider()); - let node_analysis_cache = NodeAnalysisCache::new(cache_db.node_analysis_db()); - let cjs_esm_code_analyzer = - CliCjsCodeAnalyzer::new(node_analysis_cache, self.real_fs().clone()); - let node_code_translator = Arc::new(NodeCodeTranslator::new( - cjs_esm_code_analyzer, - self.real_fs().clone(), - self.node_resolver().clone(), - self.npm_resolver().clone(), - )); - - Arc::new(NpmModuleLoader::new( - self.cjs_resolution_store().clone(), - node_code_translator, - self.real_fs(), - self.node_resolver().clone(), - )) - } - pub fn npm_fs(&self) -> Arc { let fs = self.real_fs(); create_npm_fs_resolver( diff --git a/crates/base/src/utils/graph_resolver.rs b/crates/sb_graph/graph_resolver.rs similarity index 99% rename from crates/base/src/utils/graph_resolver.rs rename to crates/sb_graph/graph_resolver.rs index c3112007d..86568cba5 100644 --- a/crates/base/src/utils/graph_resolver.rs +++ b/crates/sb_graph/graph_resolver.rs @@ -1,4 +1,4 @@ -use crate::js_worker::emitter::EmitterFactory; +use crate::EmitterFactory; use anyhow::{anyhow, bail}; use deno_core::error::AnyError; use deno_core::futures::future::LocalBoxFuture; @@ -31,7 +31,7 @@ pub enum MappedResolution { impl MappedResolution { pub fn into_specifier(self) -> Option { match self { - MappedResolution::None => Option::None, + MappedResolution::None => None, MappedResolution::PackageJson(specifier) => Some(specifier), MappedResolution::ImportMap(specifier) => Some(specifier), } diff --git a/crates/base/src/utils/graph_util.rs b/crates/sb_graph/graph_util.rs similarity index 96% rename from crates/base/src/utils/graph_util.rs rename to crates/sb_graph/graph_util.rs index c5923283f..832df0553 100644 --- a/crates/base/src/utils/graph_util.rs +++ b/crates/sb_graph/graph_util.rs @@ -1,17 +1,17 @@ -use crate::errors_rt::get_error_class_name; -use crate::js_worker::emitter::EmitterFactory; -use crate::utils::graph_resolver::CliGraphResolver; -use crate::utils::graph_util::deno_graph::ModuleError; -use crate::utils::graph_util::deno_graph::ResolutionError; +use crate::emitter::EmitterFactory; +use crate::graph_resolver::CliGraphResolver; use deno_core::error::{custom_error, AnyError}; use deno_core::parking_lot::Mutex; use deno_core::ModuleSpecifier; +use deno_graph::ModuleError; +use deno_graph::ResolutionError; use deno_semver::package::{PackageNv, PackageReq}; use eszip::deno_graph::source::Loader; use eszip::deno_graph::{GraphKind, ModuleGraph, ModuleGraphError}; use eszip::{deno_graph, EszipV2}; use module_fetcher::args::lockfile::Lockfile; use module_fetcher::cache::ParsedSourceCache; +use sb_core::errors_rt::get_error_class_name; use sb_npm::CliNpmResolver; use std::path::PathBuf; use std::sync::Arc; @@ -46,7 +46,7 @@ pub struct ModuleGraphBuilder { npm_resolver: Arc, parsed_source_cache: Arc, lockfile: Option>>, - type_check: bool, // type_checker: Arc, + type_check: bool, emitter_factory: Arc, } @@ -295,14 +295,12 @@ pub fn graph_valid( pub async fn create_eszip_from_graph_raw( graph: ModuleGraph, emitter_factory: Option>, -) -> EszipV2 { +) -> Result { let emitter = emitter_factory.unwrap_or_else(|| Arc::new(EmitterFactory::new())); let parser_arc = emitter.clone().parsed_source_cache().unwrap(); let parser = parser_arc.as_capturing_parser(); - let eszip = eszip::EszipV2::from_graph(graph, &parser, Default::default()); - - eszip.unwrap() + eszip::EszipV2::from_graph(graph, &parser, Default::default()) } pub async fn create_graph(file: PathBuf, emitter_factory: Arc) -> ModuleGraph { diff --git a/crates/sb_graph/lib.rs b/crates/sb_graph/lib.rs new file mode 100644 index 000000000..99c83af11 --- /dev/null +++ b/crates/sb_graph/lib.rs @@ -0,0 +1,74 @@ +use crate::emitter::EmitterFactory; +use crate::graph_util::{create_eszip_from_graph_raw, create_graph}; +use deno_ast::MediaType; +use deno_core::error::AnyError; +use deno_core::{serde_json, JsBuffer, ModuleSpecifier}; +use deno_fs::{FileSystem, RealFs}; +use deno_npm::NpmSystemInfo; +use eszip::EszipV2; +use sb_fs::{build_vfs, VfsOpts}; +use std::path::PathBuf; +use std::sync::Arc; + +pub mod emitter; +pub mod graph_resolver; +pub mod graph_util; + +pub const VFS_ESZIP_KEY: &str = "---SUPABASE-VFS-DATA-ESZIP---"; +pub const SOURCE_CODE_ESZIP_KEY: &str = "---SUPABASE-SOURCE-CODE-ESZIP---"; + +#[derive(Debug)] +pub enum EszipPayloadKind { + JsBufferKind(JsBuffer), + VecKind(Vec), + Eszip(EszipV2), +} + +pub async fn generate_binary_eszip( + file: PathBuf, + emitter_factory: Arc, +) -> Result { + let graph = create_graph(file.clone(), emitter_factory.clone()).await; + let eszip = create_eszip_from_graph_raw(graph, Some(emitter_factory.clone())).await; + + if let Ok(mut eszip) = eszip { + let fs_path = file.clone(); + let entry_content = RealFs.read_file_sync(fs_path.clone().as_path()).unwrap(); + let source_code: Arc = String::from_utf8(entry_content.clone())?.into(); + let emit_source = emitter_factory.emitter().unwrap().emit_parsed_source( + &ModuleSpecifier::parse("http://localhost").unwrap(), + MediaType::from_path(fs_path.clone().as_path()), + &source_code, + )?; + + let bin_code: Arc<[u8]> = emit_source.as_bytes().into(); + + let npm_res = emitter_factory.npm_resolution(); + + let (npm_vfs, _npm_files) = if npm_res.has_packages() { + let (root_dir, files) = build_vfs(VfsOpts { + npm_resolver: emitter_factory.npm_resolver().clone(), + npm_registry_api: emitter_factory.npm_api().clone(), + npm_cache: emitter_factory.npm_cache().clone(), + npm_resolution: emitter_factory.npm_resolution().clone(), + })? + .into_dir_and_files(); + + let snapshot = npm_res.serialized_valid_snapshot_for_system(&NpmSystemInfo::default()); + eszip.add_npm_snapshot(snapshot); + (Some(root_dir), files) + } else { + (None, Vec::new()) + }; + + let npm_vfs = serde_json::to_string(&npm_vfs)?.as_bytes().to_vec(); + let boxed_slice = npm_vfs.into_boxed_slice(); + + eszip.add_opaque_data(String::from(VFS_ESZIP_KEY), Arc::from(boxed_slice)); + eszip.add_opaque_data(String::from(SOURCE_CODE_ESZIP_KEY), bin_code); + + Ok(eszip) + } else { + eszip + } +} diff --git a/crates/sb_module_loader/Cargo.toml b/crates/sb_module_loader/Cargo.toml new file mode 100644 index 000000000..c2fa62579 --- /dev/null +++ b/crates/sb_module_loader/Cargo.toml @@ -0,0 +1,32 @@ +[package] +name = "sb_module_loader" +version = "0.1.0" +authors = ["Supabase "] +edition = "2021" +resolver = "2" +license = "MIT" + +[lib] +path = "lib.rs" + +[dependencies] +module_fetcher = { path = "../module_fetcher" } +sb_node = { version = "0.1.0", path = "../node" } +sb_npm = { version = "0.1.0", path = "../npm" } +sb_graph = { version = "0.1.0", path = "../sb_graph" } +sb_fs = { version = "0.1.0", path = "../sb_fs" } +sb_core = { version = "0.1.0", path = "../sb_core" } +deno_semver.workspace = true +anyhow.workspace = true +deno_core.workspace = true +eszip.workspace = true +import_map = { version = "0.15.0" } +log = { workspace = true } +serde.workspace = true +tokio.workspace = true +deno_ast.workspace = true +deno_fs.workspace = true +deno_npm.workspace = true +once_cell.workspace = true +deno_tls.workspace = true +monch.workspace = true \ No newline at end of file diff --git a/crates/sb_module_loader/lib.rs b/crates/sb_module_loader/lib.rs new file mode 100644 index 000000000..70afecb13 --- /dev/null +++ b/crates/sb_module_loader/lib.rs @@ -0,0 +1,16 @@ +use deno_core::{FastString, ModuleLoader}; +use sb_npm::CliNpmResolver; +use std::rc::Rc; +use std::sync::Arc; + +pub mod metadata; +pub mod node; +pub mod standalone; +pub mod util; + +pub struct RuntimeProviders { + pub npm_resolver: Arc, + pub module_loader: Rc, + pub fs: Arc, + pub module_code: Option, +} diff --git a/crates/sb_module_loader/metadata.rs b/crates/sb_module_loader/metadata.rs new file mode 100644 index 000000000..dafb4552f --- /dev/null +++ b/crates/sb_module_loader/metadata.rs @@ -0,0 +1,76 @@ +use deno_npm::registry::PackageDepNpmSchemeValueParseError; +use deno_semver::package::PackageReq; +use deno_semver::VersionReqSpecifierParseError; +use module_fetcher::args::package_json::{PackageJsonDepValueParseError, PackageJsonDeps}; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; + +#[derive(Serialize, Deserialize)] +enum SerializablePackageJsonDepValueParseError { + SchemeValue(String), + Specifier(String), + Unsupported { scheme: String }, +} + +impl SerializablePackageJsonDepValueParseError { + pub fn from_err(err: PackageJsonDepValueParseError) -> Self { + match err { + PackageJsonDepValueParseError::SchemeValue(err) => Self::SchemeValue(err.value), + PackageJsonDepValueParseError::Specifier(err) => { + Self::Specifier(err.source.to_string()) + } + PackageJsonDepValueParseError::Unsupported { scheme } => Self::Unsupported { scheme }, + } + } + + pub fn into_err(self) -> PackageJsonDepValueParseError { + match self { + SerializablePackageJsonDepValueParseError::SchemeValue(value) => { + PackageJsonDepValueParseError::SchemeValue(PackageDepNpmSchemeValueParseError { + value, + }) + } + SerializablePackageJsonDepValueParseError::Specifier(source) => { + PackageJsonDepValueParseError::Specifier(VersionReqSpecifierParseError { + source: monch::ParseErrorFailureError::new(source), + }) + } + SerializablePackageJsonDepValueParseError::Unsupported { scheme } => { + PackageJsonDepValueParseError::Unsupported { scheme } + } + } + } +} + +#[derive(Serialize, Deserialize)] +pub struct SerializablePackageJsonDeps( + BTreeMap>, +); + +impl SerializablePackageJsonDeps { + pub fn from_deps(deps: PackageJsonDeps) -> Self { + Self( + deps.into_iter() + .map(|(name, req)| { + let res = req.map_err(SerializablePackageJsonDepValueParseError::from_err); + (name, res) + }) + .collect(), + ) + } + + pub fn into_deps(self) -> PackageJsonDeps { + self.0 + .into_iter() + .map(|(name, res)| (name, res.map_err(|err| err.into_err()))) + .collect() + } +} + +#[derive(Deserialize, Serialize)] +pub struct Metadata { + pub ca_stores: Option>, + pub ca_data: Option>, + pub unsafely_ignore_certificate_errors: Option>, + pub package_json_deps: Option, +} diff --git a/crates/sb_module_loader/node/mod.rs b/crates/sb_module_loader/node/mod.rs new file mode 100644 index 000000000..0122bc7cf --- /dev/null +++ b/crates/sb_module_loader/node/mod.rs @@ -0,0 +1 @@ +pub mod node_module_loader; diff --git a/crates/base/src/js_worker/node_module_loader.rs b/crates/sb_module_loader/node/node_module_loader.rs similarity index 100% rename from crates/base/src/js_worker/node_module_loader.rs rename to crates/sb_module_loader/node/node_module_loader.rs diff --git a/crates/sb_module_loader/standalone/mod.rs b/crates/sb_module_loader/standalone/mod.rs new file mode 100644 index 000000000..55f9d411c --- /dev/null +++ b/crates/sb_module_loader/standalone/mod.rs @@ -0,0 +1,246 @@ +use crate::metadata::Metadata; +use crate::node::node_module_loader::{CjsResolutionStore, NpmModuleLoader}; +use crate::standalone::standalone_module_loader::{EmbeddedModuleLoader, SharedModuleLoaderState}; +use crate::RuntimeProviders; +use anyhow::Context; +use deno_core::error::AnyError; +use deno_core::url::Url; +use deno_core::{FastString, ModuleSpecifier}; +use deno_npm::NpmSystemInfo; +use deno_tls::rustls::RootCertStore; +use deno_tls::RootCertStoreProvider; +use import_map::{parse_from_json, ImportMap}; +use module_fetcher::args::package_json::PackageJsonDepsProvider; +use module_fetcher::cache::{Caches, DenoDirProvider, NodeAnalysisCache}; +use module_fetcher::file_fetcher::CacheSetting; +use module_fetcher::http_util::HttpClient; +use module_fetcher::node::CliCjsCodeAnalyzer; +use sb_core::cert::{get_root_cert_store, CaData}; +use sb_fs::file_system::DenoCompileFileSystem; +use sb_fs::load_npm_vfs; +use sb_graph::graph_resolver::MappedSpecifierResolver; +use sb_graph::{EszipPayloadKind, SOURCE_CODE_ESZIP_KEY, VFS_ESZIP_KEY}; +use sb_node::analyze::NodeCodeTranslator; +use sb_node::NodeResolver; +use sb_npm::{ + create_npm_fs_resolver, CliNpmRegistryApi, CliNpmResolver, NpmCache, NpmCacheDir, NpmResolution, +}; +use std::rc::Rc; +use std::sync::Arc; + +pub mod standalone_module_loader; + +pub struct StandaloneModuleLoaderFactory { + shared: Arc, +} + +struct StandaloneRootCertStoreProvider { + ca_stores: Option>, + ca_data: Option, + cell: once_cell::sync::OnceCell, +} + +impl RootCertStoreProvider for StandaloneRootCertStoreProvider { + fn get_or_try_init(&self) -> Result<&RootCertStore, AnyError> { + self.cell.get_or_try_init(|| { + get_root_cert_store(None, self.ca_stores.clone(), self.ca_data.clone()) + .map_err(|err| err.into()) + }) + } +} + +pub async fn create_module_loader_for_eszip( + mut eszip: eszip::EszipV2, + metadata: Metadata, + maybe_import_map: Option, +) -> Result { + // let main_module = &metadata.entrypoint; + let current_exe_path = std::env::current_exe().unwrap(); + let current_exe_name = current_exe_path.file_name().unwrap().to_string_lossy(); + let deno_dir_provider = Arc::new(DenoDirProvider::new(None)); + let root_cert_store_provider = Arc::new(StandaloneRootCertStoreProvider { + ca_stores: metadata.ca_stores, + ca_data: metadata.ca_data.map(CaData::Bytes), + cell: Default::default(), + }); + let http_client = Arc::new(HttpClient::new( + Some(root_cert_store_provider.clone()), + metadata.unsafely_ignore_certificate_errors.clone(), + )); + + // use a dummy npm registry url + let npm_registry_url = ModuleSpecifier::parse("https://localhost/").unwrap(); + let root_path = std::env::temp_dir() + .join(format!("sb-compile-{}", current_exe_name)) + .join("node_modules"); + let npm_cache_dir = NpmCacheDir::new(root_path.clone()); + + let code_fs = if let Some(module) = eszip.get_module(SOURCE_CODE_ESZIP_KEY) { + if let Some(code) = module.take_source().await { + Some(FastString::from(String::from_utf8(code.to_vec())?)) + } else { + None + } + } else { + None + }; + + let (fs, snapshot) = if let Some(snapshot) = eszip.take_npm_snapshot() { + // TODO: Support node_modules + let vfs_root_dir_path = npm_cache_dir.registry_folder(&npm_registry_url); + + let vfs_data: Vec = eszip + .get_module(VFS_ESZIP_KEY) + .unwrap() + .take_source() + .await + .unwrap() + .to_vec(); + + let vfs = load_npm_vfs(vfs_root_dir_path, &vfs_data).context("Failed to load npm vfs.")?; + + ( + Arc::new(DenoCompileFileSystem::new(vfs)) as Arc, + Some(snapshot), + ) + } else { + ( + Arc::new(deno_fs::RealFs) as Arc, + None, + ) + }; + + let npm_cache = Arc::new(NpmCache::new( + npm_cache_dir, + CacheSetting::Only, + fs.clone(), + http_client.clone(), + )); + + let npm_api = Arc::new(CliNpmRegistryApi::new( + npm_registry_url.clone(), + npm_cache.clone(), + http_client.clone(), + )); + + let npm_resolution = Arc::new(NpmResolution::from_serialized( + npm_api.clone(), + snapshot, + None, + )); + + let npm_fs_resolver = create_npm_fs_resolver( + fs.clone(), + npm_cache, + npm_registry_url, + npm_resolution.clone(), + None, + NpmSystemInfo::default(), + ); + + let npm_resolver = Arc::new(CliNpmResolver::new( + fs.clone(), + npm_resolution.clone(), + npm_fs_resolver, + None, + )); + + let node_resolver = Arc::new(NodeResolver::new(fs.clone(), npm_resolver.clone())); + let cjs_resolutions = Arc::new(CjsResolutionStore::default()); + let cache_db = Caches::new(deno_dir_provider.clone()); + let node_analysis_cache = NodeAnalysisCache::new(cache_db.node_analysis_db()); + let cjs_esm_code_analyzer = CliCjsCodeAnalyzer::new(node_analysis_cache, fs.clone()); + let node_code_translator = Arc::new(NodeCodeTranslator::new( + cjs_esm_code_analyzer, + fs.clone(), + node_resolver.clone(), + npm_resolver.clone(), + )); + let package_json_deps_provider = Arc::new(PackageJsonDepsProvider::new( + metadata + .package_json_deps + .map(|serialized| serialized.into_deps()), + )); + let maybe_import_map = maybe_import_map + .map(|import_map| Some(Arc::new(import_map))) + .unwrap_or_else(|| None); + + let module_loader_factory = StandaloneModuleLoaderFactory { + shared: Arc::new(SharedModuleLoaderState { + eszip, + mapped_specifier_resolver: MappedSpecifierResolver::new( + maybe_import_map, + package_json_deps_provider.clone(), + ), + npm_module_loader: Arc::new(NpmModuleLoader::new( + cjs_resolutions, + node_code_translator, + fs.clone(), + node_resolver.clone(), + )), + }), + }; + + Ok(RuntimeProviders { + module_loader: Rc::new(EmbeddedModuleLoader { + shared: module_loader_factory.shared.clone(), + }), + npm_resolver, + fs, + module_code: code_fs, + }) +} + +pub async fn create_module_loader_for_standalone_from_eszip_kind( + eszip_payload_kind: EszipPayloadKind, + maybe_import_map_arc: Option>, + maybe_import_map_path: Option, +) -> Result { + use deno_core::futures::io::{AllowStdIo, BufReader}; + + let eszip = match eszip_payload_kind { + EszipPayloadKind::Eszip(data) => data, + _ => { + let bytes = match eszip_payload_kind { + EszipPayloadKind::JsBufferKind(js_buffer) => Vec::from(&*js_buffer), + EszipPayloadKind::VecKind(vec) => vec, + _ => panic!("It should not get here"), + }; + + let bufreader = BufReader::new(AllowStdIo::new(bytes.as_slice())); + let (eszip, loader) = eszip::EszipV2::parse(bufreader).await.unwrap(); + + loader.await.unwrap(); + + eszip + } + }; + + let mut maybe_import_map: Option = None; + + if let Some(import_map) = maybe_import_map_arc { + let clone_import_map = (*import_map).clone(); + maybe_import_map = Some(clone_import_map); + } else if let Some(import_map_path) = maybe_import_map_path { + let import_map_url = Url::parse(import_map_path.as_str())?; + if let Some(import_map_module) = eszip.get_import_map(import_map_url.as_str()) { + if let Some(source) = import_map_module.source().await { + let source = std::str::from_utf8(&source)?.to_string(); + let result = parse_from_json(&import_map_url, &source)?; + maybe_import_map = Some(result.import_map); + } + } + } + + create_module_loader_for_eszip( + eszip, + Metadata { + ca_stores: None, + ca_data: None, + unsafely_ignore_certificate_errors: None, + package_json_deps: None, + }, + maybe_import_map, + ) + .await +} diff --git a/crates/base/src/js_worker/standalone.rs b/crates/sb_module_loader/standalone/standalone_module_loader.rs similarity index 66% rename from crates/base/src/js_worker/standalone.rs rename to crates/sb_module_loader/standalone/standalone_module_loader.rs index e3f30380d..97a070ddc 100644 --- a/crates/base/src/js_worker/standalone.rs +++ b/crates/sb_module_loader/standalone/standalone_module_loader.rs @@ -1,38 +1,32 @@ -use crate::deno_runtime::RuntimeProviders; -use crate::js_worker::emitter::EmitterFactory; -use crate::js_worker::node_module_loader::NpmModuleLoader; -use crate::standalone::binary::Metadata; -use crate::standalone::create_module_loader_for_eszip; -use crate::utils::graph_resolver::MappedSpecifierResolver; +// Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. + +use crate::node::node_module_loader::NpmModuleLoader; use deno_ast::MediaType; -use deno_core::error::{generic_error, type_error, AnyError}; +use deno_core::error::generic_error; +use deno_core::error::type_error; +use deno_core::error::AnyError; use deno_core::futures::FutureExt; -use deno_core::{ModuleLoader, ModuleSpecifier, ModuleType, ResolutionKind}; +use deno_core::ModuleLoader; +use deno_core::ModuleSpecifier; +use deno_core::ModuleType; +use deno_core::ResolutionKind; use deno_semver::npm::NpmPackageReqReference; -use import_map::ImportMap; use module_fetcher::file_fetcher::get_source_from_data_url; -use sb_eszip::module_loader::EszipPayloadKind; use std::pin::Pin; use std::sync::Arc; +use crate::util::arc_u8_to_arc_str; +use sb_graph::graph_resolver::MappedSpecifierResolver; + pub struct SharedModuleLoaderState { - eszip: eszip::EszipV2, - mapped_specifier_resolver: MappedSpecifierResolver, - npm_module_loader: Arc, + pub(crate) eszip: eszip::EszipV2, + pub(crate) mapped_specifier_resolver: MappedSpecifierResolver, + pub(crate) npm_module_loader: Arc, } #[derive(Clone)] pub struct EmbeddedModuleLoader { - shared: Arc, -} - -fn arc_u8_to_arc_str(arc_u8: Arc<[u8]>) -> Result, std::str::Utf8Error> { - // Check that the string is valid UTF-8. - std::str::from_utf8(&arc_u8)?; - // SAFETY: the string is valid UTF-8, and the layout Arc<[u8]> is the same as - // Arc. This is proven by the From> impl for Arc<[u8]> from the - // standard library. - Ok(unsafe { std::mem::transmute(arc_u8) }) + pub(crate) shared: Arc, } impl ModuleLoader for EmbeddedModuleLoader { @@ -98,6 +92,7 @@ impl ModuleLoader for EmbeddedModuleLoader { _is_dynamic: bool, ) -> Pin> { let is_data_uri = get_source_from_data_url(original_specifier).ok(); + let permissions = sb_node::allow_all(); if let Some((source, _)) = is_data_uri { return Box::pin(deno_core::futures::future::ready(Ok( deno_core::ModuleSource::new( @@ -108,7 +103,6 @@ impl ModuleLoader for EmbeddedModuleLoader { ))); } - let permissions = sb_node::allow_all(); if let Some(result) = self.shared.npm_module_loader.load_sync_if_in_npm_package( original_specifier, maybe_referrer, @@ -166,53 +160,3 @@ impl ModuleLoader for EmbeddedModuleLoader { .boxed_local() } } - -pub fn create_shared_state_for_module_loader( - mut eszip: eszip::EszipV2, - maybe_import_map: Option>, -) -> SharedModuleLoaderState { - let mut emitter = EmitterFactory::new(); - - if let Some(snapshot) = eszip.take_npm_snapshot() { - emitter.set_npm_snapshot(Some(snapshot)); - } - - let shared_module_state = SharedModuleLoaderState { - eszip, - mapped_specifier_resolver: MappedSpecifierResolver::new( - maybe_import_map, - emitter.package_json_deps_provider().clone(), - ), - npm_module_loader: emitter.npm_module_loader(), - }; - shared_module_state -} - -pub async fn create_module_loader_for_standalone_from_eszip_kind( - eszip_payload_kind: EszipPayloadKind, - maybe_import_map: Option, -) -> RuntimeProviders { - use deno_core::futures::io::{AllowStdIo, BufReader}; - let bytes = match eszip_payload_kind { - EszipPayloadKind::JsBufferKind(js_buffer) => Vec::from(&*js_buffer), - EszipPayloadKind::VecKind(vec) => vec, - }; - - let bufreader = BufReader::new(AllowStdIo::new(bytes.as_slice())); - let (eszip, loader) = eszip::EszipV2::parse(bufreader).await.unwrap(); - - loader.await.unwrap(); - - create_module_loader_for_eszip( - eszip, - Metadata { - ca_stores: None, - ca_data: None, - unsafely_ignore_certificate_errors: None, - package_json_deps: None, - }, - maybe_import_map, - ) - .await - .unwrap() -} diff --git a/crates/sb_module_loader/util.rs b/crates/sb_module_loader/util.rs new file mode 100644 index 000000000..172719bfb --- /dev/null +++ b/crates/sb_module_loader/util.rs @@ -0,0 +1,10 @@ +use std::sync::Arc; + +pub fn arc_u8_to_arc_str(arc_u8: Arc<[u8]>) -> Result, std::str::Utf8Error> { + // Check that the string is valid UTF-8. + std::str::from_utf8(&arc_u8)?; + // SAFETY: the string is valid UTF-8, and the layout Arc<[u8]> is the same as + // Arc. This is proven by the From> impl for Arc<[u8]> from the + // standard library. + Ok(unsafe { std::mem::transmute(arc_u8) }) +} diff --git a/crates/sb_os/Cargo.toml b/crates/sb_os/Cargo.toml index e4ee0c3f8..563e6ae0d 100644 --- a/crates/sb_os/Cargo.toml +++ b/crates/sb_os/Cargo.toml @@ -14,5 +14,4 @@ path = "lib.rs" deno_core.workspace = true libc.workspace = true serde.workspace = true -sb_worker_context = { version = "0.1.0", path = "../sb_worker_context" } sb_core = { version = "0.1.0", path = "../sb_core" } diff --git a/crates/sb_worker_context/Cargo.toml b/crates/sb_worker_context/Cargo.toml deleted file mode 100644 index a83577748..000000000 --- a/crates/sb_worker_context/Cargo.toml +++ /dev/null @@ -1,22 +0,0 @@ -[package] -name = "sb_worker_context" -version = "0.1.0" -authors = ["Supabase "] -edition = "2021" -resolver = "2" -description = "We'll take care of this later" -license = "MIT" - -[lib] -path = "lib.rs" - -[dependencies] -anyhow = { workspace = true } -deno_core.workspace = true -event_worker ={ version = "0.1.0", path = "../event_worker" } -enum-as-inner = "0.6.0" -hyper.workspace = true -serde.workspace = true -sb_eszip = { version = "0.1.0", path = "../sb_eszip" } -tokio.workspace = true -uuid.workspace = true diff --git a/crates/sb_worker_context/lib.rs b/crates/sb_worker_context/lib.rs deleted file mode 100644 index 6e2d56295..000000000 --- a/crates/sb_worker_context/lib.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod essentials; diff --git a/crates/sb_workers/Cargo.toml b/crates/sb_workers/Cargo.toml index 9bf964b02..f0fd76ae2 100644 --- a/crates/sb_workers/Cargo.toml +++ b/crates/sb_workers/Cargo.toml @@ -17,9 +17,9 @@ deno_core.workspace = true tokio.workspace = true deno_http.workspace = true hyper.workspace = true -sb_eszip = { version = "0.1.0", path = "../sb_eszip" } serde.workspace = true bytes.workspace = true log.workspace = true -sb_worker_context = { version = "0.1.0", path = "../sb_worker_context" } event_worker ={ version = "0.1.0", path = "../event_worker" } +sb_graph ={ version = "0.1.0", path = "../sb_graph" } +enum-as-inner = "0.6.0" \ No newline at end of file diff --git a/crates/sb_worker_context/essentials.rs b/crates/sb_workers/context.rs similarity index 98% rename from crates/sb_worker_context/essentials.rs rename to crates/sb_workers/context.rs index 8a87d459e..38d6e3777 100644 --- a/crates/sb_worker_context/essentials.rs +++ b/crates/sb_workers/context.rs @@ -8,7 +8,7 @@ use std::path::PathBuf; use tokio::sync::{mpsc, oneshot}; use uuid::Uuid; -use sb_eszip::module_loader::EszipPayloadKind; +use sb_graph::EszipPayloadKind; #[derive(Debug, Clone)] pub struct UserWorkerRuntimeOpts { diff --git a/crates/sb_workers/lib.rs b/crates/sb_workers/lib.rs index 83cb78878..422c00675 100644 --- a/crates/sb_workers/lib.rs +++ b/crates/sb_workers/lib.rs @@ -1,3 +1,9 @@ +pub mod context; + +use crate::context::{ + CreateUserWorkerResult, UserWorkerMsgs, UserWorkerRuntimeOpts, WorkerContextInitOpts, + WorkerRuntimeOpts, +}; use anyhow::Error; use deno_core::error::{custom_error, type_error, AnyError}; use deno_core::futures::stream::Peekable; @@ -10,11 +16,7 @@ use deno_core::{ use hyper::body::HttpBody; use hyper::header::{HeaderName, HeaderValue}; use hyper::{Body, Request, Response}; -use sb_eszip::module_loader::EszipPayloadKind; -use sb_worker_context::essentials::{ - CreateUserWorkerResult, UserWorkerMsgs, UserWorkerRuntimeOpts, WorkerContextInitOpts, - WorkerRuntimeOpts, -}; +use sb_graph::EszipPayloadKind; use serde::{Deserialize, Serialize}; use std::borrow::Cow; use std::cell::RefCell;