From a365598fbf0fb1e6f03e24fac674dff8c8226537 Mon Sep 17 00:00:00 2001 From: Divy Srivastava Date: Wed, 1 May 2024 20:21:47 +0530 Subject: [PATCH 1/8] fix(node): seperate worker module cache --- cli/module_loader.rs | 62 ++++++++++++++++++- cli/resolver.rs | 1 + .../node/worker_threads_cache/__test__.jsonc | 4 ++ .../specs/node/worker_threads_cache/main.out | 2 + tests/specs/node/worker_threads_cache/main.ts | 13 ++++ 5 files changed, 81 insertions(+), 1 deletion(-) create mode 100644 tests/specs/node/worker_threads_cache/__test__.jsonc create mode 100644 tests/specs/node/worker_threads_cache/main.out create mode 100644 tests/specs/node/worker_threads_cache/main.ts diff --git a/cli/module_loader.rs b/cli/module_loader.rs index a6c8d133810d0..ddf83f7f52cfe 100644 --- a/cli/module_loader.rs +++ b/cli/module_loader.rs @@ -93,6 +93,20 @@ impl ModuleLoadPreparer { } } + pub fn new_for_worker( + &self, + graph_container: Arc, + ) -> Self { + Self { + options: self.options.clone(), + graph_container, + lockfile: self.lockfile.clone(), + module_graph_builder: self.module_graph_builder.clone(), + progress_bar: self.progress_bar.clone(), + type_checker: self.type_checker.clone(), + } + } + /// This method must be called for a module or a static importer of that /// module before attempting to `load()` it from a `JsRuntime`. It will /// populate the graph data in memory with the necessary source code, write @@ -221,6 +235,7 @@ impl ModuleLoadPreparer { } } +#[derive(Clone)] struct PreparedModuleLoader { emitter: Arc, graph_container: Arc, @@ -228,6 +243,14 @@ struct PreparedModuleLoader { } impl PreparedModuleLoader { + pub fn for_worker(&self, graph_container: Arc) -> Self { + Self { + emitter: self.emitter.clone(), + graph_container, + parsed_source_cache: self.parsed_source_cache.clone(), + } + } + pub fn load_prepared_module( &self, specifier: &ModuleSpecifier, @@ -319,6 +342,34 @@ struct SharedCliModuleLoaderState { module_info_cache: Arc, } +impl SharedCliModuleLoaderState { + fn for_worker(&self) -> Self { + let graph_container = + Arc::new(ModuleGraphContainer::new(deno_graph::GraphKind::All)); + let module_load_preparer = self + .module_load_preparer + .new_for_worker(graph_container.clone()); + let prepared_module_loader = self + .prepared_module_loader + .for_worker(graph_container.clone()); + + Self { + lib_window: self.lib_worker, + lib_worker: self.lib_worker, + is_inspecting: self.is_inspecting, + is_repl: self.is_repl, + graph_container, + module_load_preparer: Arc::new(module_load_preparer), + prepared_module_loader, + resolver: self.resolver.clone(), + node_resolver: self.node_resolver.clone(), + npm_module_loader: self.npm_module_loader.clone(), + code_cache: self.code_cache.clone(), + module_info_cache: self.module_info_cache.clone(), + } + } +} + pub struct CliModuleLoaderFactory { shared: Arc, } @@ -367,12 +418,19 @@ impl CliModuleLoaderFactory { lib: TsTypeLib, root_permissions: PermissionsContainer, dynamic_permissions: PermissionsContainer, + worker: bool, ) -> Rc { + let shared = if worker { + Arc::new(self.shared.for_worker()) + } else { + self.shared.clone() + }; + Rc::new(CliModuleLoader { lib, root_permissions, dynamic_permissions, - shared: self.shared.clone(), + shared, }) } } @@ -387,6 +445,7 @@ impl ModuleLoaderFactory for CliModuleLoaderFactory { self.shared.lib_window, root_permissions, dynamic_permissions, + false, ) } @@ -399,6 +458,7 @@ impl ModuleLoaderFactory for CliModuleLoaderFactory { self.shared.lib_worker, root_permissions, dynamic_permissions, + true, ) } diff --git a/cli/resolver.rs b/cli/resolver.rs index 32233e961f82a..38b54150bc15e 100644 --- a/cli/resolver.rs +++ b/cli/resolver.rs @@ -249,6 +249,7 @@ impl CliNodeResolver { } } +#[derive(Clone)] pub struct NpmModuleLoader { cjs_resolutions: Arc, node_code_translator: Arc, diff --git a/tests/specs/node/worker_threads_cache/__test__.jsonc b/tests/specs/node/worker_threads_cache/__test__.jsonc new file mode 100644 index 0000000000000..fc471ddcef8cc --- /dev/null +++ b/tests/specs/node/worker_threads_cache/__test__.jsonc @@ -0,0 +1,4 @@ +{ + "args": "run -A main.ts", + "out": "main.out" +} diff --git a/tests/specs/node/worker_threads_cache/main.out b/tests/specs/node/worker_threads_cache/main.out new file mode 100644 index 0000000000000..d14c028e5b751 --- /dev/null +++ b/tests/specs/node/worker_threads_cache/main.out @@ -0,0 +1,2 @@ +[Module: null prototype] { default: true } +[Module: null prototype] { default: false } diff --git a/tests/specs/node/worker_threads_cache/main.ts b/tests/specs/node/worker_threads_cache/main.ts new file mode 100644 index 0000000000000..2dd258a009e7f --- /dev/null +++ b/tests/specs/node/worker_threads_cache/main.ts @@ -0,0 +1,13 @@ +import fs from 'node:fs/promises'; +import { Worker, isMainThread } from 'node:worker_threads'; + +await fs.writeFile('mod.mjs', 'export default ' + isMainThread); + +const path = new URL('mod.mjs', import.meta.url); +const i = await import(path.href); +console.log(i); + +if (isMainThread) { + const worker = new Worker(new URL('test.mjs', import.meta.url)); + worker.on('message', (msg) => console.log(msg)); +} From b1b35bd43b7ca33ad7a687ad04f8f289940c2441 Mon Sep 17 00:00:00 2001 From: Divy Srivastava Date: Wed, 1 May 2024 20:26:05 +0530 Subject: [PATCH 2/8] fmt --- tests/specs/node/worker_threads_cache/main.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/specs/node/worker_threads_cache/main.ts b/tests/specs/node/worker_threads_cache/main.ts index 2dd258a009e7f..65a4dd239d175 100644 --- a/tests/specs/node/worker_threads_cache/main.ts +++ b/tests/specs/node/worker_threads_cache/main.ts @@ -1,13 +1,13 @@ -import fs from 'node:fs/promises'; -import { Worker, isMainThread } from 'node:worker_threads'; +import fs from "node:fs/promises"; +import { isMainThread, Worker } from "node:worker_threads"; -await fs.writeFile('mod.mjs', 'export default ' + isMainThread); +await fs.writeFile("mod.mjs", "export default " + isMainThread); -const path = new URL('mod.mjs', import.meta.url); +const path = new URL("mod.mjs", import.meta.url); const i = await import(path.href); console.log(i); if (isMainThread) { - const worker = new Worker(new URL('test.mjs', import.meta.url)); - worker.on('message', (msg) => console.log(msg)); + const worker = new Worker(new URL("test.mjs", import.meta.url)); + worker.on("message", (msg) => console.log(msg)); } From 2dfeacffdf84a075da2e374b9a224bf898dc57af Mon Sep 17 00:00:00 2001 From: David Sherret Date: Fri, 3 May 2024 19:34:43 -0400 Subject: [PATCH 3/8] Initial refactor, but I'm going to change this so that dynamic imports are shared across bench/test workers. --- cli/args/mod.rs | 9 + cli/factory.rs | 48 ++-- cli/graph_util.rs | 62 ----- cli/lsp/testing/execution.rs | 8 +- cli/main.rs | 16 +- cli/module_load_preparer.rs | 206 ++++++++++++++++ cli/resolver.rs | 16 +- cli/standalone/mod.rs | 41 ++-- cli/tools/bench/mod.rs | 44 ++-- cli/tools/installer.rs | 7 +- cli/tools/jupyter/mod.rs | 4 + cli/tools/repl/mod.rs | 2 + cli/tools/run/mod.rs | 30 ++- cli/tools/test/mod.rs | 51 ++-- cli/worker.rs | 40 +-- cli/workers/mod.rs | 6 + cli/workers/module_graph_container.rs | 65 +++++ cli/{ => workers}/module_loader.rs | 338 +++++--------------------- 18 files changed, 513 insertions(+), 480 deletions(-) create mode 100644 cli/module_load_preparer.rs create mode 100644 cli/workers/mod.rs create mode 100644 cli/workers/module_graph_container.rs rename cli/{ => workers}/module_loader.rs (70%) diff --git a/cli/args/mod.rs b/cli/args/mod.rs index f4d5743dce199..24fd0610b11ad 100644 --- a/cli/args/mod.rs +++ b/cli/args/mod.rs @@ -12,6 +12,7 @@ use self::package_json::PackageJsonDeps; use ::import_map::ImportMap; use deno_ast::SourceMapOption; use deno_core::resolve_url_or_path; +use deno_graph::GraphKind; use deno_npm::resolution::ValidSerializedNpmResolutionSnapshot; use deno_npm::NpmSystemInfo; use deno_runtime::deno_tls::RootCertStoreProvider; @@ -870,6 +871,14 @@ impl CliOptions { self.maybe_config_file.as_ref().map(|f| f.specifier.clone()) } + pub fn graph_kind(&self) -> GraphKind { + match self.sub_command() { + DenoSubcommand::Cache(_) => GraphKind::All, + DenoSubcommand::Check(_) => GraphKind::TypesOnly, + _ => self.type_check_mode().as_graph_kind(), + } + } + pub fn ts_type_lib_window(&self) -> TsTypeLib { TsTypeLib::DenoWindow } diff --git a/cli/factory.rs b/cli/factory.rs index bfaf96f39cdfe..ff56e453b5b91 100644 --- a/cli/factory.rs +++ b/cli/factory.rs @@ -23,11 +23,10 @@ use crate::emit::Emitter; use crate::file_fetcher::FileFetcher; use crate::graph_util::FileWatcherReporter; use crate::graph_util::ModuleGraphBuilder; -use crate::graph_util::ModuleGraphContainer; use crate::graph_util::ModuleGraphCreator; use crate::http_util::HttpClient; -use crate::module_loader::CliModuleLoaderFactory; -use crate::module_loader::ModuleLoadPreparer; +use crate::module_load_preparer::MainModuleGraphPreparer; +use crate::module_load_preparer::ModuleLoadPreparer; use crate::node::CliCjsCodeAnalyzer; use crate::node::CliNodeCodeTranslator; use crate::npm::create_cli_npm_resolver; @@ -53,6 +52,7 @@ use crate::util::progress_bar::ProgressBar; use crate::util::progress_bar::ProgressBarStyle; use crate::worker::CliMainWorkerFactory; use crate::worker::CliMainWorkerOptions; +use crate::workers::CliModuleLoaderFactory; use std::path::PathBuf; use deno_core::error::AnyError; @@ -60,7 +60,6 @@ use deno_core::futures::FutureExt; use deno_core::parking_lot::Mutex; use deno_core::FeatureChecker; -use deno_graph::GraphKind; use deno_lockfile::WorkspaceMemberConfig; use deno_runtime::deno_fs; use deno_runtime::deno_node::analyze::NodeCodeTranslator; @@ -157,7 +156,6 @@ struct CliFactoryServices { emit_cache: Deferred, emitter: Deferred>, fs: Deferred>, - graph_container: Deferred>, lockfile: Deferred>>>, maybe_import_map: Deferred>>, maybe_inspector_server: Deferred>>, @@ -672,19 +670,6 @@ impl CliFactory { .await } - pub fn graph_container(&self) -> &Arc { - self.services.graph_container.get_or_init(|| { - let graph_kind = match self.options.sub_command() { - // todo(dsherret): ideally the graph container would not be used - // for deno cache because it doesn't dynamically load modules - DenoSubcommand::Cache(_) => GraphKind::All, - DenoSubcommand::Check(_) => GraphKind::TypesOnly, - _ => self.options.type_check_mode().as_graph_kind(), - }; - Arc::new(ModuleGraphContainer::new(graph_kind)) - }) - } - pub fn maybe_inspector_server( &self, ) -> Result<&Option>, AnyError> { @@ -705,7 +690,6 @@ impl CliFactory { .get_or_try_init_async(async { Ok(Arc::new(ModuleLoadPreparer::new( self.options.clone(), - self.graph_container().clone(), self.maybe_lockfile().clone(), self.module_graph_builder().await?.clone(), self.text_only_progress_bar().clone(), @@ -769,6 +753,15 @@ impl CliFactory { )) } + pub async fn create_main_module_graph_preparer( + &self, + ) -> Result { + Ok(MainModuleGraphPreparer::new( + self.options.clone(), + self.module_load_preparer().await?.clone(), + )) + } + pub async fn create_cli_main_worker_factory( &self, ) -> Result { @@ -790,11 +783,14 @@ impl CliFactory { self.blob_store().clone(), Box::new(CliModuleLoaderFactory::new( &self.options, + if self.options.code_cache_enabled() { + Some(self.code_cache()?.clone()) + } else { + None + }, self.emitter()?.clone(), - self.graph_container().clone(), + self.module_info_cache()?.clone(), self.module_load_preparer().await?.clone(), - self.parsed_source_cache().clone(), - self.resolver().await?.clone(), cli_node_resolver.clone(), NpmModuleLoader::new( self.cjs_resolutions().clone(), @@ -802,12 +798,8 @@ impl CliFactory { fs.clone(), cli_node_resolver.clone(), ), - if self.options.code_cache_enabled() { - Some(self.code_cache()?.clone()) - } else { - None - }, - self.module_info_cache()?.clone(), + self.parsed_source_cache().clone(), + self.resolver().await?.clone(), )), self.root_cert_store_provider().clone(), self.fs().clone(), diff --git a/cli/graph_util.rs b/cli/graph_util.rs index ac7f8a3652650..20d90ab3f1dcf 100644 --- a/cli/graph_util.rs +++ b/cli/graph_util.rs @@ -18,8 +18,6 @@ use crate::tools::check; use crate::tools::check::TypeChecker; use crate::util::file_watcher::WatcherCommunicator; use crate::util::fs::canonicalize_path; -use crate::util::sync::TaskQueue; -use crate::util::sync::TaskQueuePermit; use deno_runtime::fs_util::specifier_to_file_path; use deno_config::WorkspaceMemberConfig; @@ -27,7 +25,6 @@ use deno_core::anyhow::bail; use deno_core::error::custom_error; use deno_core::error::AnyError; use deno_core::parking_lot::Mutex; -use deno_core::parking_lot::RwLock; use deno_core::ModuleSpecifier; use deno_graph::source::Loader; use deno_graph::source::ResolutionMode; @@ -763,40 +760,6 @@ fn get_resolution_error_bare_specifier( } } -/// Holds the `ModuleGraph` and what parts of it are type checked. -pub struct ModuleGraphContainer { - // Allow only one request to update the graph data at a time, - // but allow other requests to read from it at any time even - // while another request is updating the data. - update_queue: Arc, - inner: Arc>>, -} - -impl ModuleGraphContainer { - pub fn new(graph_kind: GraphKind) -> Self { - Self { - update_queue: Default::default(), - inner: Arc::new(RwLock::new(Arc::new(ModuleGraph::new(graph_kind)))), - } - } - - /// Acquires a permit to modify the module graph without other code - /// having the chance to modify it. In the meantime, other code may - /// still read from the existing module graph. - pub async fn acquire_update_permit(&self) -> ModuleGraphUpdatePermit { - let permit = self.update_queue.acquire().await; - ModuleGraphUpdatePermit { - permit, - inner: self.inner.clone(), - graph: (**self.inner.read()).clone(), - } - } - - pub fn graph(&self) -> Arc { - self.inner.read().clone() - } -} - /// Gets if any of the specified root's "file:" dependents are in the /// provided changed set. pub fn has_graph_root_local_dependent_changed( @@ -829,31 +792,6 @@ pub fn has_graph_root_local_dependent_changed( false } -/// A permit for updating the module graph. When complete and -/// everything looks fine, calling `.commit()` will store the -/// new graph in the ModuleGraphContainer. -pub struct ModuleGraphUpdatePermit<'a> { - permit: TaskQueuePermit<'a>, - inner: Arc>>, - graph: ModuleGraph, -} - -impl<'a> ModuleGraphUpdatePermit<'a> { - /// Gets the module graph for mutation. - pub fn graph_mut(&mut self) -> &mut ModuleGraph { - &mut self.graph - } - - /// Saves the mutated module graph in the container - /// and returns an Arc to the new module graph. - pub fn commit(self) -> Arc { - let graph = Arc::new(self.graph); - *self.inner.write() = graph.clone(); - drop(self.permit); // explicit drop for clarity - graph - } -} - #[derive(Clone, Debug)] pub struct FileWatcherReporter { watcher_communicator: Arc, diff --git a/cli/lsp/testing/execution.rs b/cli/lsp/testing/execution.rs index 73916d0c2081d..8252e15728c8c 100644 --- a/cli/lsp/testing/execution.rs +++ b/cli/lsp/testing/execution.rs @@ -219,10 +219,11 @@ impl TestRun { // file would have impact on other files, which is undesirable. let permissions = Permissions::from_options(&factory.cli_options().permissions_options())?; + let mut main_graph_preparer = + factory.create_main_module_graph_preparer().await?; test::check_specifiers( - factory.cli_options(), factory.file_fetcher()?, - factory.module_load_preparer().await?, + &mut main_graph_preparer, self .queue .iter() @@ -263,6 +264,7 @@ impl TestRun { let mut test_steps = IndexMap::new(); let worker_factory = Arc::new(factory.create_cli_main_worker_factory().await?); + let module_graph = Arc::new(main_graph_preparer.into_graph()); let join_handles = queue.into_iter().map(move |specifier| { let specifier = specifier.clone(); @@ -284,6 +286,7 @@ impl TestRun { .unwrap_or_default(), }; let token = self.token.clone(); + let module_graph = module_graph.clone(); spawn_blocking(move || { if fail_fast_tracker.should_stop() { @@ -298,6 +301,7 @@ impl TestRun { worker_factory, permissions, specifier, + module_graph, worker_sender, fail_fast_tracker, test::TestSpecifierOptions { diff --git a/cli/main.rs b/cli/main.rs index 3b103e7807b56..60c917fd8ea1d 100644 --- a/cli/main.rs +++ b/cli/main.rs @@ -13,7 +13,7 @@ mod http_util; mod js; mod jsr; mod lsp; -mod module_loader; +mod module_load_preparer; mod napi; mod node; mod npm; @@ -25,6 +25,7 @@ mod tsc; mod util; mod version; mod worker; +mod workers; use crate::args::flags_from_vec; use crate::args::DenoSubcommand; @@ -112,18 +113,19 @@ async fn run_subcommand(flags: Flags) -> Result { }), DenoSubcommand::Cache(cache_flags) => spawn_subcommand(async move { let factory = CliFactory::from_flags(flags)?; - let module_load_preparer = factory.module_load_preparer().await?; let emitter = factory.emitter()?; - let graph_container = factory.graph_container(); - module_load_preparer + let mut module_graph_preparer = + factory.create_main_module_graph_preparer().await?; + module_graph_preparer .load_and_type_check_files(&cache_flags.files) .await?; - emitter.cache_module_emits(&graph_container.graph()) + emitter.cache_module_emits(&module_graph_preparer.graph()) }), DenoSubcommand::Check(check_flags) => spawn_subcommand(async move { let factory = CliFactory::from_flags(flags)?; - let module_load_preparer = factory.module_load_preparer().await?; - module_load_preparer + let mut module_graph_preparer = + factory.create_main_module_graph_preparer().await?; + module_graph_preparer .load_and_type_check_files(&check_flags.files) .await }), diff --git a/cli/module_load_preparer.rs b/cli/module_load_preparer.rs new file mode 100644 index 0000000000000..4016cc5edb89c --- /dev/null +++ b/cli/module_load_preparer.rs @@ -0,0 +1,206 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +use crate::args::CliOptions; +use crate::args::TsTypeLib; +use crate::graph_util::graph_lock_or_exit; +use crate::graph_util::CreateGraphOptions; +use crate::graph_util::ModuleGraphBuilder; +use crate::tools::check; +use crate::tools::check::TypeChecker; +use crate::util::progress_bar::ProgressBar; + +use deno_core::anyhow::Context; +use deno_core::error::AnyError; +use deno_core::parking_lot::Mutex; +use deno_core::resolve_url_or_path; +use deno_core::ModuleSpecifier; +use deno_graph::ModuleGraph; +use deno_lockfile::Lockfile; +use deno_runtime::permissions::PermissionsContainer; +use deno_terminal::colors; +use std::sync::Arc; + +pub struct ModuleLoadPreparer { + options: Arc, + lockfile: Option>>, + module_graph_builder: Arc, + progress_bar: ProgressBar, + type_checker: Arc, +} + +impl ModuleLoadPreparer { + #[allow(clippy::too_many_arguments)] + pub fn new( + options: Arc, + lockfile: Option>>, + module_graph_builder: Arc, + progress_bar: ProgressBar, + type_checker: Arc, + ) -> Self { + Self { + options, + lockfile, + module_graph_builder, + progress_bar, + type_checker, + } + } + + /// This method must be called for a module or a static importer of that + /// module before attempting to `load()` it from a `JsRuntime`. It will + /// populate the graph data in memory with the necessary source code, write + /// emits where necessary or report any module graph / type checking errors. + #[allow(clippy::too_many_arguments)] + pub async fn prepare_module_load( + &self, + graph: &mut ModuleGraph, + roots: &[ModuleSpecifier], + is_dynamic: bool, + lib: TsTypeLib, + permissions: PermissionsContainer, + ) -> Result<(), AnyError> { + log::debug!("Preparing module load."); + let _pb_clear_guard = self.progress_bar.clear_guard(); + + let mut cache = self.module_graph_builder.create_fetch_cacher(permissions); + log::debug!("Building module graph."); + let has_type_checked = !graph.roots.is_empty(); + + self + .module_graph_builder + .build_graph_with_npm_resolution( + graph, + CreateGraphOptions { + is_dynamic, + graph_kind: graph.graph_kind(), + roots: roots.to_vec(), + loader: Some(&mut cache), + }, + ) + .await?; + + self.module_graph_builder.graph_roots_valid(graph, &roots)?; + + // If there is a lockfile... + if let Some(lockfile) = &self.lockfile { + let mut lockfile = lockfile.lock(); + // validate the integrity of all the modules + graph_lock_or_exit(graph, &mut lockfile); + // update it with anything new + lockfile.write().context("Failed writing lockfile.")?; + } + + drop(_pb_clear_guard); + + // type check if necessary + if self.options.type_check_mode().is_true() && !has_type_checked { + self + .type_checker + .check( + // todo(perf): since this is only done the first time the graph is + // created, we could avoid the clone of the graph here by providing + // the actual graph on the first run and then getting the Arc + // back from the return value. + graph.clone(), + check::CheckOptions { + build_fast_check_graph: true, + lib, + log_ignored_options: false, + reload: self.options.reload_flag(), + type_check_mode: self.options.type_check_mode(), + }, + ) + .await?; + } + + log::debug!("Prepared module load."); + + Ok(()) + } +} + +pub struct MainModuleGraphPreparer { + options: Arc, + module_load_preparer: Arc, + graph: ModuleGraph, +} + +impl MainModuleGraphPreparer { + pub fn new( + options: Arc, + module_load_preparer: Arc, + ) -> Self { + Self { + graph: ModuleGraph::new(options.graph_kind()), + options, + module_load_preparer, + } + } + + pub fn into_graph(self) -> ModuleGraph { + self.graph + } + + pub fn graph(&self) -> &ModuleGraph { + &self.graph + } + + pub async fn check_specifiers( + &mut self, + specifiers: &[ModuleSpecifier], + ) -> Result<(), AnyError> { + let lib = self.options.ts_type_lib_window(); + self + .module_load_preparer + .prepare_module_load( + &mut self.graph, + specifiers, + false, + lib, + PermissionsContainer::allow_all(), + ) + .await?; + Ok(()) + } + + /// Helper around prepare_module_load that loads and type checks + /// the provided files. + pub async fn load_and_type_check_files( + &mut self, + files: &[String], + ) -> Result<(), AnyError> { + let specifiers = self.collect_specifiers(files)?; + + if specifiers.is_empty() { + log::warn!("{} No matching files found.", colors::yellow("Warning")); + } + + self.check_specifiers(&specifiers).await + } + + fn collect_specifiers( + &self, + files: &[String], + ) -> Result, AnyError> { + let excludes = self.options.resolve_config_excludes()?; + Ok( + files + .iter() + .filter_map(|file| { + let file_url = + resolve_url_or_path(file, self.options.initial_cwd()).ok()?; + if file_url.scheme() != "file" { + return Some(file_url); + } + // ignore local files that match any of files listed in `exclude` option + let file_path = file_url.to_file_path().ok()?; + if excludes.matches_path(&file_path) { + None + } else { + Some(file_url) + } + }) + .collect::>(), + ) + } +} diff --git a/cli/resolver.rs b/cli/resolver.rs index 38b54150bc15e..18c3c765f6339 100644 --- a/cli/resolver.rs +++ b/cli/resolver.rs @@ -91,8 +91,8 @@ impl CliNodeResolver { } } - pub fn in_npm_package(&self, referrer: &ModuleSpecifier) -> bool { - self.npm_resolver.in_npm_package(referrer) + pub fn in_npm_package(&self, specifier: &ModuleSpecifier) -> bool { + self.npm_resolver.in_npm_package(specifier) } pub fn get_closest_package_json( @@ -272,18 +272,6 @@ impl NpmModuleLoader { } } - pub fn maybe_prepare_load( - &self, - specifier: &ModuleSpecifier, - ) -> Option> { - if self.node_resolver.in_npm_package(specifier) { - // nothing to prepare - Some(Ok(())) - } else { - None - } - } - pub fn load_sync_if_in_npm_package( &self, specifier: &ModuleSpecifier, diff --git a/cli/standalone/mod.rs b/cli/standalone/mod.rs index 3cfeb4f4ce58c..0eb34c1d8e06c 100644 --- a/cli/standalone/mod.rs +++ b/cli/standalone/mod.rs @@ -27,6 +27,7 @@ use crate::util::progress_bar::ProgressBarStyle; use crate::util::v8::construct_v8_flags; use crate::worker::CliMainWorkerFactory; use crate::worker::CliMainWorkerOptions; +use crate::worker::ModuleLoaderAndSourceMapGetter; use crate::worker::ModuleLoaderFactory; use deno_ast::MediaType; use deno_core::anyhow::Context; @@ -42,6 +43,7 @@ use deno_core::ModuleSpecifier; use deno_core::ModuleType; use deno_core::RequestedModuleType; use deno_core::ResolutionKind; +use deno_graph::ModuleGraph; use deno_runtime::deno_fs; use deno_runtime::deno_node::analyze::NodeCodeTranslator; use deno_runtime::deno_node::NodeResolutionMode; @@ -280,32 +282,33 @@ struct StandaloneModuleLoaderFactory { impl ModuleLoaderFactory for StandaloneModuleLoaderFactory { fn create_for_main( &self, + _module_graph: Arc, root_permissions: PermissionsContainer, dynamic_permissions: PermissionsContainer, - ) -> Rc { - Rc::new(EmbeddedModuleLoader { - shared: self.shared.clone(), - root_permissions, - dynamic_permissions, - }) + ) -> ModuleLoaderAndSourceMapGetter { + ModuleLoaderAndSourceMapGetter { + module_loader: Rc::new(EmbeddedModuleLoader { + shared: self.shared.clone(), + root_permissions, + dynamic_permissions, + }), + source_map_getter: None, + } } fn create_for_worker( &self, root_permissions: PermissionsContainer, dynamic_permissions: PermissionsContainer, - ) -> Rc { - Rc::new(EmbeddedModuleLoader { - shared: self.shared.clone(), - root_permissions, - dynamic_permissions, - }) - } - - fn create_source_map_getter( - &self, - ) -> Option> { - None + ) -> ModuleLoaderAndSourceMapGetter { + ModuleLoaderAndSourceMapGetter { + module_loader: Rc::new(EmbeddedModuleLoader { + shared: self.shared.clone(), + root_permissions, + dynamic_permissions, + }), + source_map_getter: None, + } } } @@ -594,6 +597,8 @@ pub async fn run( .create_main_worker( WorkerExecutionMode::Run, main_module.clone(), + // todo(THIS PR): remove this + Arc::new(ModuleGraph::new(deno_graph::GraphKind::All)), permissions, ) .await?; diff --git a/cli/tools/bench/mod.rs b/cli/tools/bench/mod.rs index e2411ed125f50..0214c9db885f7 100644 --- a/cli/tools/bench/mod.rs +++ b/cli/tools/bench/mod.rs @@ -8,7 +8,6 @@ use crate::display::write_json_to_stdout; use crate::factory::CliFactory; use crate::factory::CliFactoryBuilder; use crate::graph_util::has_graph_root_local_dependent_changed; -use crate::module_loader::ModuleLoadPreparer; use crate::ops; use crate::tools::test::format_test_error; use crate::tools::test::TestFilter; @@ -32,6 +31,7 @@ use deno_core::unsync::spawn_blocking; use deno_core::v8; use deno_core::ModuleSpecifier; use deno_core::PollEventLoopOptions; +use deno_graph::ModuleGraph; use deno_runtime::permissions::Permissions; use deno_runtime::permissions::PermissionsContainer; use deno_runtime::tokio_util::create_and_run_current_thread; @@ -145,29 +145,12 @@ fn create_reporter( Box::new(ConsoleReporter::new(show_output)) } -/// Type check a collection of module and document specifiers. -async fn check_specifiers( - cli_options: &CliOptions, - module_load_preparer: &ModuleLoadPreparer, - specifiers: Vec, -) -> Result<(), AnyError> { - let lib = cli_options.ts_type_lib_window(); - module_load_preparer - .prepare_module_load( - specifiers, - false, - lib, - PermissionsContainer::allow_all(), - ) - .await?; - Ok(()) -} - /// Run a single specifier as an executable bench module. async fn bench_specifier( worker_factory: Arc, permissions: Permissions, specifier: ModuleSpecifier, + main_module_graph: Arc, sender: UnboundedSender, filter: TestFilter, ) -> Result<(), AnyError> { @@ -175,6 +158,7 @@ async fn bench_specifier( worker_factory, permissions, specifier.clone(), + main_module_graph, &sender, filter, ) @@ -200,6 +184,7 @@ async fn bench_specifier_inner( worker_factory: Arc, permissions: Permissions, specifier: ModuleSpecifier, + main_module_graph: Arc, sender: &UnboundedSender, filter: TestFilter, ) -> Result<(), AnyError> { @@ -207,6 +192,7 @@ async fn bench_specifier_inner( .create_custom_worker( WorkerExecutionMode::Bench, specifier.clone(), + main_module_graph, PermissionsContainer::new(permissions), vec![ops::bench::deno_bench::init_ops(sender.clone())], Default::default(), @@ -287,6 +273,7 @@ async fn bench_specifiers( worker_factory: Arc, permissions: &Permissions, specifiers: Vec, + main_module_graph: Arc, options: BenchSpecifierOptions, ) -> Result<(), AnyError> { let (sender, mut receiver) = unbounded_channel::(); @@ -298,11 +285,13 @@ async fn bench_specifiers( let permissions = permissions.clone(); let sender = sender.clone(); let options = option_for_handles.clone(); + let main_module_graph = main_module_graph.clone(); spawn_blocking(move || { let future = bench_specifier( worker_factory, permissions, specifier, + main_module_graph, sender, options.filter, ); @@ -445,12 +434,9 @@ pub async fn run_benchmarks( return Err(generic_error("No bench modules found")); } - check_specifiers( - cli_options, - factory.module_load_preparer().await?, - specifiers.clone(), - ) - .await?; + let mut main_graph_preparer = + factory.create_main_module_graph_preparer().await?; + main_graph_preparer.check_specifiers(&specifiers).await?; if bench_options.no_run { return Ok(()); @@ -463,6 +449,7 @@ pub async fn run_benchmarks( worker_factory, &permissions, specifiers, + Arc::new(main_graph_preparer.into_graph()), BenchSpecifierOptions { filter: TestFilter::from_flag(&bench_options.filter), json: bench_options.json, @@ -507,7 +494,6 @@ pub async fn run_benchmarks_with_watch( let graph_kind = cli_options.type_check_mode().as_graph_kind(); let module_graph_creator = factory.module_graph_creator().await?; - let module_load_preparer = factory.module_load_preparer().await?; let bench_modules = collect_specifiers( bench_options.files.clone(), @@ -559,8 +545,9 @@ pub async fn run_benchmarks_with_watch( .filter(|specifier| bench_modules_to_reload.contains(specifier)) .collect::>(); - check_specifiers(cli_options, module_load_preparer, specifiers.clone()) - .await?; + let mut main_graph_preparer = + factory.create_main_module_graph_preparer().await?; + main_graph_preparer.check_specifiers(&specifiers).await?; if bench_options.no_run { return Ok(()); @@ -571,6 +558,7 @@ pub async fn run_benchmarks_with_watch( worker_factory, &permissions, specifiers, + Arc::new(main_graph_preparer.into_graph()), BenchSpecifierOptions { filter: TestFilter::from_flag(&bench_options.filter), json: bench_options.json, diff --git a/cli/tools/installer.rs b/cli/tools/installer.rs index 6d9c1294e1c3f..515b15f2156de 100644 --- a/cli/tools/installer.rs +++ b/cli/tools/installer.rs @@ -267,9 +267,10 @@ pub async fn install_command( }; // ensure the module is cached - CliFactory::from_flags(flags.clone())? - .module_load_preparer() - .await? + let factory = CliFactory::from_flags(flags.clone())?; + let mut module_graph_preparer = + factory.create_main_module_graph_preparer().await?; + module_graph_preparer .load_and_type_check_files(&[install_flags_global.module_url.clone()]) .await?; diff --git a/cli/tools/jupyter/mod.rs b/cli/tools/jupyter/mod.rs index da1c4bc4d8f13..32bf9680e06b5 100644 --- a/cli/tools/jupyter/mod.rs +++ b/cli/tools/jupyter/mod.rs @@ -1,5 +1,7 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. +use std::sync::Arc; + use crate::args::Flags; use crate::args::JupyterFlags; use crate::ops; @@ -18,6 +20,7 @@ use deno_core::resolve_url_or_path; use deno_core::serde::Deserialize; use deno_core::serde_json; use deno_core::url::Url; +use deno_graph::ModuleGraph; use deno_runtime::deno_io::Stdio; use deno_runtime::deno_io::StdioPipe; use deno_runtime::permissions::Permissions; @@ -91,6 +94,7 @@ pub async fn kernel( .create_custom_worker( WorkerExecutionMode::Jupyter, main_module.clone(), + Arc::new(ModuleGraph::new(cli_options.graph_kind())), permissions, vec![ ops::jupyter::deno_jupyter::init_ops(stdio_tx.clone()), diff --git a/cli/tools/repl/mod.rs b/cli/tools/repl/mod.rs index d1c1cab7137e8..7763c97d26d3e 100644 --- a/cli/tools/repl/mod.rs +++ b/cli/tools/repl/mod.rs @@ -13,6 +13,7 @@ use deno_core::error::AnyError; use deno_core::futures::StreamExt; use deno_core::serde_json; use deno_core::unsync::spawn_blocking; +use deno_graph::ModuleGraph; use deno_runtime::permissions::Permissions; use deno_runtime::permissions::PermissionsContainer; use deno_runtime::WorkerExecutionMode; @@ -173,6 +174,7 @@ pub async fn run(flags: Flags, repl_flags: ReplFlags) -> Result { .create_custom_worker( WorkerExecutionMode::Repl, main_module.clone(), + Arc::new(ModuleGraph::new(cli_options.graph_kind())), permissions, vec![crate::ops::testing::deno_test::init_ops(test_event_sender)], Default::default(), diff --git a/cli/tools/run/mod.rs b/cli/tools/run/mod.rs index 9f4bfeb96431c..51b1dbf9816b7 100644 --- a/cli/tools/run/mod.rs +++ b/cli/tools/run/mod.rs @@ -1,8 +1,10 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. use std::io::Read; +use std::sync::Arc; use deno_core::error::AnyError; +use deno_graph::ModuleGraph; use deno_runtime::permissions::Permissions; use deno_runtime::permissions::PermissionsContainer; use deno_runtime::WorkerExecutionMode; @@ -69,7 +71,12 @@ To grant permissions, set them before the script argument. For example: )?); let worker_factory = factory.create_cli_main_worker_factory().await?; let mut worker = worker_factory - .create_main_worker(mode, main_module, permissions) + .create_main_worker( + mode, + main_module, + Arc::new(ModuleGraph::new(cli_options.graph_kind())), + permissions, + ) .await?; let exit_code = worker.run().await?; @@ -99,7 +106,12 @@ pub async fn run_from_stdin(flags: Flags) -> Result { }); let mut worker = worker_factory - .create_main_worker(WorkerExecutionMode::Run, main_module, permissions) + .create_main_worker( + WorkerExecutionMode::Run, + main_module, + Arc::new(ModuleGraph::new(cli_options.graph_kind())), + permissions, + ) .await?; let exit_code = worker.run().await?; Ok(exit_code) @@ -137,7 +149,12 @@ async fn run_with_watch( let mut worker = factory .create_cli_main_worker_factory() .await? - .create_main_worker(mode, main_module, permissions) + .create_main_worker( + mode, + main_module, + Arc::new(ModuleGraph::new(cli_options.graph_kind())), + permissions, + ) .await?; if watch_flags.hmr { @@ -186,7 +203,12 @@ pub async fn eval_command( )?); let worker_factory = factory.create_cli_main_worker_factory().await?; let mut worker = worker_factory - .create_main_worker(WorkerExecutionMode::Eval, main_module, permissions) + .create_main_worker( + WorkerExecutionMode::Eval, + main_module, + Arc::new(ModuleGraph::new(cli_options.graph_kind())), + permissions, + ) .await?; let exit_code = worker.run().await?; Ok(exit_code) diff --git a/cli/tools/test/mod.rs b/cli/tools/test/mod.rs index ffa0fef9eec4a..4d04bd3c81e41 100644 --- a/cli/tools/test/mod.rs +++ b/cli/tools/test/mod.rs @@ -11,7 +11,7 @@ use crate::factory::CliFactoryBuilder; use crate::file_fetcher::File; use crate::file_fetcher::FileFetcher; use crate::graph_util::has_graph_root_local_dependent_changed; -use crate::module_loader::ModuleLoadPreparer; +use crate::module_load_preparer::MainModuleGraphPreparer; use crate::ops; use crate::util::file_watcher; use crate::util::fs::collect_specifiers; @@ -52,6 +52,7 @@ use deno_core::v8; use deno_core::ModuleSpecifier; use deno_core::OpState; use deno_core::PollEventLoopOptions; +use deno_graph::ModuleGraph; use deno_runtime::deno_io::Stdio; use deno_runtime::deno_io::StdioPipe; use deno_runtime::fmt_errors::format_js_error; @@ -578,6 +579,7 @@ fn get_test_reporter(options: &TestSpecifiersOptions) -> Box { async fn configure_main_worker( worker_factory: Arc, specifier: &Url, + module_graph: Arc, permissions: Permissions, worker_sender: TestEventWorkerSender, options: &TestSpecifierOptions, @@ -586,6 +588,7 @@ async fn configure_main_worker( .create_custom_worker( WorkerExecutionMode::Test, specifier.clone(), + module_graph, PermissionsContainer::new(permissions), vec![ops::testing::deno_test::init_ops(worker_sender.sender)], Stdio { @@ -631,6 +634,7 @@ pub async fn test_specifier( worker_factory: Arc, permissions: Permissions, specifier: ModuleSpecifier, + module_graph: Arc, worker_sender: TestEventWorkerSender, fail_fast_tracker: FailFastTracker, options: TestSpecifierOptions, @@ -641,6 +645,7 @@ pub async fn test_specifier( let (coverage_collector, mut worker) = configure_main_worker( worker_factory, &specifier, + module_graph, permissions, worker_sender, &options, @@ -1305,12 +1310,10 @@ async fn fetch_inline_files( /// Type check a collection of module and document specifiers. pub async fn check_specifiers( - cli_options: &CliOptions, file_fetcher: &FileFetcher, - module_load_preparer: &ModuleLoadPreparer, + main_graph_preparer: &mut MainModuleGraphPreparer, specifiers: Vec<(ModuleSpecifier, TestMode)>, ) -> Result<(), AnyError> { - let lib = cli_options.ts_type_lib_window(); let inline_files = fetch_inline_files( file_fetcher, specifiers @@ -1330,20 +1333,13 @@ pub async fn check_specifiers( let specifiers = inline_files .iter() .map(|file| file.specifier.clone()) - .collect(); + .collect::>(); for file in inline_files { file_fetcher.insert_memory_files(file); } - module_load_preparer - .prepare_module_load( - specifiers, - false, - lib, - PermissionsContainer::new(Permissions::allow_all()), - ) - .await?; + main_graph_preparer.check_specifiers(&specifiers).await?; } let module_specifiers = specifiers @@ -1355,15 +1351,10 @@ pub async fn check_specifiers( None } }) - .collect(); + .collect::>(); - module_load_preparer - .prepare_module_load( - module_specifiers, - false, - lib, - PermissionsContainer::allow_all(), - ) + main_graph_preparer + .check_specifiers(&module_specifiers) .await?; Ok(()) @@ -1376,6 +1367,7 @@ async fn test_specifiers( worker_factory: Arc, permissions: &Permissions, specifiers: Vec, + module_graph: Arc, options: TestSpecifiersOptions, ) -> Result<(), AnyError> { let specifiers = if let Some(seed) = options.specifier.shuffle { @@ -1406,11 +1398,13 @@ async fn test_specifiers( let worker_sender = test_event_sender_factory.worker(); let fail_fast_tracker = fail_fast_tracker.clone(); let specifier_options = options.specifier.clone(); + let module_graph = module_graph.clone(); spawn_blocking(move || { create_and_run_current_thread(test_specifier( worker_factory, permissions, specifier, + module_graph, worker_sender, fail_fast_tracker, specifier_options, @@ -1710,7 +1704,6 @@ pub async fn run_tests( let cli_options = factory.cli_options(); let test_options = cli_options.resolve_test_options(test_flags)?; let file_fetcher = factory.file_fetcher()?; - let module_load_preparer = factory.module_load_preparer().await?; // Various test files should not share the same permissions in terms of // `PermissionsContainer` - otherwise granting/revoking permissions in one // file would have impact on other files, which is undesirable. @@ -1730,10 +1723,12 @@ pub async fn run_tests( return Err(generic_error("No test modules found")); } + let mut main_graph_preparer = + factory.create_main_module_graph_preparer().await?; + check_specifiers( - cli_options, file_fetcher, - module_load_preparer, + &mut main_graph_preparer, specifiers_with_mode.clone(), ) .await?; @@ -1755,6 +1750,7 @@ pub async fn run_tests( _ => Some(s), }) .collect(), + Arc::new(main_graph_preparer.into_graph()), TestSpecifiersOptions { cwd: Url::from_directory_path(cli_options.initial_cwd()).map_err( |_| { @@ -1872,7 +1868,6 @@ pub async fn run_tests_with_watch( let worker_factory = Arc::new(factory.create_cli_main_worker_factory().await?); - let module_load_preparer = factory.module_load_preparer().await?; let specifiers_with_mode = fetch_specifiers_with_test_mode( &cli_options, file_fetcher, @@ -1884,10 +1879,11 @@ pub async fn run_tests_with_watch( .filter(|(specifier, _)| test_modules_to_reload.contains(specifier)) .collect::>(); + let mut main_graph_preparer = + factory.create_main_module_graph_preparer().await?; check_specifiers( - &cli_options, file_fetcher, - module_load_preparer, + &mut main_graph_preparer, specifiers_with_mode.clone(), ) .await?; @@ -1906,6 +1902,7 @@ pub async fn run_tests_with_watch( _ => Some(s), }) .collect(), + Arc::new(main_graph_preparer.into_graph()), TestSpecifiersOptions { cwd: Url::from_directory_path(cli_options.initial_cwd()).map_err( |_| { diff --git a/cli/worker.rs b/cli/worker.rs index edc4ef907a481..ecee96a6222c5 100644 --- a/cli/worker.rs +++ b/cli/worker.rs @@ -22,6 +22,7 @@ use deno_core::ModuleLoader; use deno_core::PollEventLoopOptions; use deno_core::SharedArrayBufferStore; use deno_core::SourceMapGetter; +use deno_graph::ModuleGraph; use deno_lockfile::Lockfile; use deno_runtime::code_cache; use deno_runtime::deno_broadcast_channel::InMemoryBroadcastChannel; @@ -58,20 +59,24 @@ use crate::util::file_watcher::WatcherCommunicator; use crate::util::file_watcher::WatcherRestartMode; use crate::version; +pub struct ModuleLoaderAndSourceMapGetter { + pub module_loader: Rc, + pub source_map_getter: Option>, +} + pub trait ModuleLoaderFactory: Send + Sync { fn create_for_main( &self, + main_module_graph: Arc, root_permissions: PermissionsContainer, dynamic_permissions: PermissionsContainer, - ) -> Rc; + ) -> ModuleLoaderAndSourceMapGetter; fn create_for_worker( &self, root_permissions: PermissionsContainer, dynamic_permissions: PermissionsContainer, - ) -> Rc; - - fn create_source_map_getter(&self) -> Option>; + ) -> ModuleLoaderAndSourceMapGetter; } #[async_trait::async_trait(?Send)] @@ -454,12 +459,14 @@ impl CliMainWorkerFactory { &self, mode: WorkerExecutionMode, main_module: ModuleSpecifier, + main_module_graph: Arc, permissions: PermissionsContainer, ) -> Result { self .create_custom_worker( mode, main_module, + main_module_graph, permissions, vec![], Default::default(), @@ -471,6 +478,7 @@ impl CliMainWorkerFactory { &self, mode: WorkerExecutionMode, main_module: ModuleSpecifier, + main_module_graph: Arc, permissions: PermissionsContainer, custom_extensions: Vec, stdio: deno_runtime::deno_io::Stdio, @@ -548,11 +556,14 @@ impl CliMainWorkerFactory { (main_module, false) }; - let module_loader = shared - .module_loader_factory - .create_for_main(PermissionsContainer::allow_all(), permissions.clone()); - let maybe_source_map_getter = - shared.module_loader_factory.create_source_map_getter(); + let ModuleLoaderAndSourceMapGetter { + module_loader, + source_map_getter, + } = shared.module_loader_factory.create_for_main( + main_module_graph, + PermissionsContainer::allow_all(), + permissions.clone(), + ); let maybe_inspector_server = shared.maybe_inspector_server.clone(); let create_web_worker_cb = @@ -624,7 +635,7 @@ impl CliMainWorkerFactory { .clone(), root_cert_store_provider: Some(shared.root_cert_store_provider.clone()), seed: shared.options.seed, - source_map_getter: maybe_source_map_getter, + source_map_getter, format_js_error_fn: Some(Arc::new(format_js_error)), create_web_worker_cb, maybe_inspector_server, @@ -766,12 +777,13 @@ fn create_web_worker_callback( Arc::new(move |args| { let maybe_inspector_server = shared.maybe_inspector_server.clone(); - let module_loader = shared.module_loader_factory.create_for_worker( + let ModuleLoaderAndSourceMapGetter { + module_loader, + source_map_getter, + } = shared.module_loader_factory.create_for_worker( args.parent_permissions.clone(), args.permissions.clone(), ); - let maybe_source_map_getter = - shared.module_loader_factory.create_source_map_getter(); let create_web_worker_cb = create_web_worker_callback(mode, shared.clone(), stdio.clone()); @@ -834,7 +846,7 @@ fn create_web_worker_callback( seed: shared.options.seed, create_web_worker_cb, format_js_error_fn: Some(Arc::new(format_js_error)), - source_map_getter: maybe_source_map_getter, + source_map_getter, module_loader, fs: shared.fs.clone(), npm_resolver: Some(shared.npm_resolver.clone().into_npm_resolver()), diff --git a/cli/workers/mod.rs b/cli/workers/mod.rs new file mode 100644 index 0000000000000..d17f0755b6ca3 --- /dev/null +++ b/cli/workers/mod.rs @@ -0,0 +1,6 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +mod module_graph_container; +mod module_loader; + +pub use module_loader::CliModuleLoaderFactory; diff --git a/cli/workers/module_graph_container.rs b/cli/workers/module_graph_container.rs new file mode 100644 index 0000000000000..363c1bd15c782 --- /dev/null +++ b/cli/workers/module_graph_container.rs @@ -0,0 +1,65 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +use std::cell::RefCell; +use std::rc::Rc; +use std::sync::Arc; + +use deno_core::unsync::TaskQueue; +use deno_core::unsync::TaskQueuePermit; +use deno_graph::ModuleGraph; + +/// Holds the `ModuleGraph`. +pub struct WorkerModuleGraphContainer { + // Allow only one request to update the graph data at a time, + // but allow other requests to read from it at any time even + // while another request is updating the data. + update_queue: Rc, + inner: Rc>>, +} + +impl WorkerModuleGraphContainer { + pub fn new(module_graph: Arc) -> Self { + Self { + update_queue: Default::default(), + inner: Rc::new(RefCell::new(module_graph)), + } + } + + /// Acquires a permit to modify the module graph without other code + /// having the chance to modify it. In the meantime, other code may + /// still read from the existing module graph. + pub async fn acquire_update_permit(&self) -> WorkerModuleGraphUpdatePermit { + let permit = self.update_queue.acquire().await; + WorkerModuleGraphUpdatePermit { + permit, + inner: self.inner.clone(), + graph: (**self.inner.borrow()).clone(), + } + } + + pub fn graph(&self) -> Arc { + self.inner.borrow().clone() + } +} + +/// A permit for updating the module graph. When complete and +/// everything looks fine, calling `.commit()` will store the +/// new graph in the ModuleGraphContainer. +pub struct WorkerModuleGraphUpdatePermit { + permit: TaskQueuePermit, + inner: Rc>>, + graph: ModuleGraph, +} + +impl WorkerModuleGraphUpdatePermit { + /// Gets the module graph for mutation. + pub fn graph_mut(&mut self) -> &mut ModuleGraph { + &mut self.graph + } + + /// Saves the mutated module graph in the container. + pub fn commit(self) { + *self.inner.borrow_mut() = Arc::new(self.graph); + drop(self.permit); // explicit drop for clarity + } +} diff --git a/cli/module_loader.rs b/cli/workers/module_loader.rs similarity index 70% rename from cli/module_loader.rs rename to cli/workers/module_loader.rs index ddf83f7f52cfe..f90bbf34fd311 100644 --- a/cli/module_loader.rs +++ b/cli/workers/module_loader.rs @@ -8,20 +8,15 @@ use crate::cache::CodeCache; use crate::cache::ModuleInfoCache; use crate::cache::ParsedSourceCache; use crate::emit::Emitter; -use crate::graph_util::graph_lock_or_exit; -use crate::graph_util::CreateGraphOptions; -use crate::graph_util::ModuleGraphBuilder; -use crate::graph_util::ModuleGraphContainer; +use crate::module_load_preparer::ModuleLoadPreparer; use crate::node; use crate::resolver::CliGraphResolver; use crate::resolver::CliNodeResolver; use crate::resolver::ModuleCodeStringSource; use crate::resolver::NpmModuleLoader; -use crate::tools::check; -use crate::tools::check::TypeChecker; -use crate::util::progress_bar::ProgressBar; use crate::util::text_encoding::code_without_source_map; use crate::util::text_encoding::source_map_from_code; +use crate::worker::ModuleLoaderAndSourceMapGetter; use crate::worker::ModuleLoaderFactory; use deno_ast::MediaType; @@ -33,9 +28,7 @@ use deno_core::error::generic_error; use deno_core::error::AnyError; use deno_core::futures::future::FutureExt; use deno_core::futures::Future; -use deno_core::parking_lot::Mutex; use deno_core::resolve_url; -use deno_core::resolve_url_or_path; use deno_core::ModuleCodeString; use deno_core::ModuleLoader; use deno_core::ModuleSource; @@ -47,210 +40,33 @@ use deno_core::ResolutionKind; use deno_core::SourceMapGetter; use deno_graph::source::ResolutionMode; use deno_graph::source::Resolver; +use deno_graph::GraphKind; use deno_graph::JsModule; use deno_graph::JsonModule; use deno_graph::Module; +use deno_graph::ModuleGraph; use deno_graph::Resolution; -use deno_lockfile::Lockfile; use deno_runtime::code_cache; use deno_runtime::deno_node::NodeResolutionMode; use deno_runtime::fs_util::code_timestamp; use deno_runtime::permissions::PermissionsContainer; use deno_semver::npm::NpmPackageReqReference; -use deno_terminal::colors; use std::borrow::Cow; use std::pin::Pin; use std::rc::Rc; use std::str; use std::sync::Arc; -pub struct ModuleLoadPreparer { - options: Arc, - graph_container: Arc, - lockfile: Option>>, - module_graph_builder: Arc, - progress_bar: ProgressBar, - type_checker: Arc, -} - -impl ModuleLoadPreparer { - #[allow(clippy::too_many_arguments)] - pub fn new( - options: Arc, - graph_container: Arc, - lockfile: Option>>, - module_graph_builder: Arc, - progress_bar: ProgressBar, - type_checker: Arc, - ) -> Self { - Self { - options, - graph_container, - lockfile, - module_graph_builder, - progress_bar, - type_checker, - } - } - - pub fn new_for_worker( - &self, - graph_container: Arc, - ) -> Self { - Self { - options: self.options.clone(), - graph_container, - lockfile: self.lockfile.clone(), - module_graph_builder: self.module_graph_builder.clone(), - progress_bar: self.progress_bar.clone(), - type_checker: self.type_checker.clone(), - } - } - - /// This method must be called for a module or a static importer of that - /// module before attempting to `load()` it from a `JsRuntime`. It will - /// populate the graph data in memory with the necessary source code, write - /// emits where necessary or report any module graph / type checking errors. - #[allow(clippy::too_many_arguments)] - pub async fn prepare_module_load( - &self, - roots: Vec, - is_dynamic: bool, - lib: TsTypeLib, - permissions: PermissionsContainer, - ) -> Result<(), AnyError> { - log::debug!("Preparing module load."); - let _pb_clear_guard = self.progress_bar.clear_guard(); - - let mut cache = self.module_graph_builder.create_fetch_cacher(permissions); - log::debug!("Creating module graph."); - let mut graph_update_permit = - self.graph_container.acquire_update_permit().await; - let graph = graph_update_permit.graph_mut(); - let has_type_checked = !graph.roots.is_empty(); - - self - .module_graph_builder - .build_graph_with_npm_resolution( - graph, - CreateGraphOptions { - is_dynamic, - graph_kind: graph.graph_kind(), - roots: roots.clone(), - loader: Some(&mut cache), - }, - ) - .await?; - - self.module_graph_builder.graph_roots_valid(graph, &roots)?; - - // If there is a lockfile... - if let Some(lockfile) = &self.lockfile { - let mut lockfile = lockfile.lock(); - // validate the integrity of all the modules - graph_lock_or_exit(graph, &mut lockfile); - // update it with anything new - lockfile.write().context("Failed writing lockfile.")?; - } - - // save the graph and get a reference to the new graph - let graph = graph_update_permit.commit(); - - drop(_pb_clear_guard); - - // type check if necessary - if self.options.type_check_mode().is_true() && !has_type_checked { - self - .type_checker - .check( - // todo(perf): since this is only done the first time the graph is - // created, we could avoid the clone of the graph here by providing - // the actual graph on the first run and then getting the Arc - // back from the return value. - (*graph).clone(), - check::CheckOptions { - build_fast_check_graph: true, - lib, - log_ignored_options: false, - reload: self.options.reload_flag(), - type_check_mode: self.options.type_check_mode(), - }, - ) - .await?; - } - - log::debug!("Prepared module load."); - - Ok(()) - } - - /// Helper around prepare_module_load that loads and type checks - /// the provided files. - pub async fn load_and_type_check_files( - &self, - files: &[String], - ) -> Result<(), AnyError> { - let lib = self.options.ts_type_lib_window(); - - let specifiers = self.collect_specifiers(files)?; - - if specifiers.is_empty() { - log::warn!("{} No matching files found.", colors::yellow("Warning")); - } - - self - .prepare_module_load( - specifiers, - false, - lib, - PermissionsContainer::allow_all(), - ) - .await - } - - fn collect_specifiers( - &self, - files: &[String], - ) -> Result, AnyError> { - let excludes = self.options.resolve_config_excludes()?; - Ok( - files - .iter() - .filter_map(|file| { - let file_url = - resolve_url_or_path(file, self.options.initial_cwd()).ok()?; - if file_url.scheme() != "file" { - return Some(file_url); - } - // ignore local files that match any of files listed in `exclude` option - let file_path = file_url.to_file_path().ok()?; - if excludes.matches_path(&file_path) { - None - } else { - Some(file_url) - } - }) - .collect::>(), - ) - } -} +use super::module_graph_container::WorkerModuleGraphContainer; #[derive(Clone)] struct PreparedModuleLoader { emitter: Arc, - graph_container: Arc, + graph_container: Rc, parsed_source_cache: Arc, } impl PreparedModuleLoader { - pub fn for_worker(&self, graph_container: Arc) -> Self { - Self { - emitter: self.emitter.clone(), - graph_container, - parsed_source_cache: self.parsed_source_cache.clone(), - } - } - pub fn load_prepared_module( &self, specifier: &ModuleSpecifier, @@ -328,46 +144,19 @@ impl PreparedModuleLoader { } struct SharedCliModuleLoaderState { + graph_kind: GraphKind, lib_window: TsTypeLib, lib_worker: TsTypeLib, is_inspecting: bool, is_repl: bool, - graph_container: Arc, - module_load_preparer: Arc, - prepared_module_loader: PreparedModuleLoader, + code_cache: Option>, + emitter: Arc, resolver: Arc, + module_info_cache: Arc, + module_load_preparer: Arc, node_resolver: Arc, npm_module_loader: NpmModuleLoader, - code_cache: Option>, - module_info_cache: Arc, -} - -impl SharedCliModuleLoaderState { - fn for_worker(&self) -> Self { - let graph_container = - Arc::new(ModuleGraphContainer::new(deno_graph::GraphKind::All)); - let module_load_preparer = self - .module_load_preparer - .new_for_worker(graph_container.clone()); - let prepared_module_loader = self - .prepared_module_loader - .for_worker(graph_container.clone()); - - Self { - lib_window: self.lib_worker, - lib_worker: self.lib_worker, - is_inspecting: self.is_inspecting, - is_repl: self.is_repl, - graph_container, - module_load_preparer: Arc::new(module_load_preparer), - prepared_module_loader, - resolver: self.resolver.clone(), - node_resolver: self.node_resolver.clone(), - npm_module_loader: self.npm_module_loader.clone(), - code_cache: self.code_cache.clone(), - module_info_cache: self.module_info_cache.clone(), - } - } + parsed_source_cache: Arc, } pub struct CliModuleLoaderFactory { @@ -378,18 +167,18 @@ impl CliModuleLoaderFactory { #[allow(clippy::too_many_arguments)] pub fn new( options: &CliOptions, + code_cache: Option>, emitter: Arc, - graph_container: Arc, + module_info_cache: Arc, module_load_preparer: Arc, - parsed_source_cache: Arc, - resolver: Arc, node_resolver: Arc, npm_module_loader: NpmModuleLoader, - code_cache: Option>, - module_info_cache: Arc, + parsed_source_cache: Arc, + resolver: Arc, ) -> Self { Self { shared: Arc::new(SharedCliModuleLoaderState { + graph_kind: options.graph_kind(), lib_window: options.ts_type_lib_window(), lib_worker: options.ts_type_lib_worker(), is_inspecting: options.is_inspecting(), @@ -397,55 +186,58 @@ impl CliModuleLoaderFactory { options.sub_command(), DenoSubcommand::Repl(_) | DenoSubcommand::Jupyter(_) ), - prepared_module_loader: PreparedModuleLoader { - emitter, - graph_container: graph_container.clone(), - parsed_source_cache, - }, - graph_container, + code_cache, + emitter, + module_info_cache, module_load_preparer, - resolver, node_resolver, npm_module_loader, - code_cache, - module_info_cache, + parsed_source_cache, + resolver, }), } } fn create_with_lib( &self, + module_graph: Arc, lib: TsTypeLib, root_permissions: PermissionsContainer, dynamic_permissions: PermissionsContainer, - worker: bool, - ) -> Rc { - let shared = if worker { - Arc::new(self.shared.for_worker()) - } else { - self.shared.clone() - }; - - Rc::new(CliModuleLoader { + ) -> ModuleLoaderAndSourceMapGetter { + let graph_container = + Rc::new(WorkerModuleGraphContainer::new(module_graph)); + let loader = Rc::new(CliModuleLoader { lib, root_permissions, dynamic_permissions, - shared, - }) + graph_container: graph_container.clone(), + prepared_module_loader: PreparedModuleLoader { + emitter: self.shared.emitter.clone(), + graph_container, + parsed_source_cache: self.shared.parsed_source_cache.clone(), + }, + shared: self.shared.clone(), + }); + ModuleLoaderAndSourceMapGetter { + module_loader: loader.clone(), + source_map_getter: Some(loader), + } } } impl ModuleLoaderFactory for CliModuleLoaderFactory { fn create_for_main( &self, + starting_module_graph: Arc, root_permissions: PermissionsContainer, dynamic_permissions: PermissionsContainer, - ) -> Rc { + ) -> ModuleLoaderAndSourceMapGetter { self.create_with_lib( + starting_module_graph, self.shared.lib_window, root_permissions, dynamic_permissions, - false, ) } @@ -453,20 +245,15 @@ impl ModuleLoaderFactory for CliModuleLoaderFactory { &self, root_permissions: PermissionsContainer, dynamic_permissions: PermissionsContainer, - ) -> Rc { + ) -> ModuleLoaderAndSourceMapGetter { self.create_with_lib( + // create a fresh module graph for the worker + Arc::new(ModuleGraph::new(self.shared.graph_kind)), self.shared.lib_worker, root_permissions, dynamic_permissions, - true, ) } - - fn create_source_map_getter(&self) -> Option> { - Some(Rc::new(CliSourceMapGetter { - shared: self.shared.clone(), - })) - } } struct CliModuleLoader { @@ -479,6 +266,8 @@ struct CliModuleLoader { /// "root permissions" for Web Worker. dynamic_permissions: PermissionsContainer, shared: Arc, + graph_container: Rc, + prepared_module_loader: PreparedModuleLoader, } impl CliModuleLoader { @@ -502,7 +291,6 @@ impl CliModuleLoader { result? } else { self - .shared .prepared_module_loader .load_prepared_module(specifier, maybe_referrer)? }; @@ -606,7 +394,7 @@ impl CliModuleLoader { }; } - let graph = self.shared.graph_container.graph(); + let graph = self.graph_container.graph(); let maybe_resolved = match graph.get(referrer) { Some(Module::Js(module)) => { module.dependencies.get(specifier).map(|d| &d.maybe_code) @@ -772,13 +560,12 @@ impl ModuleLoader for CliModuleLoader { _maybe_referrer: Option, is_dynamic: bool, ) -> Pin>>> { - if let Some(result) = - self.shared.npm_module_loader.maybe_prepare_load(specifier) - { - return Box::pin(deno_core::futures::future::ready(result)); + if self.shared.node_resolver.in_npm_package(&specifier) { + return Box::pin(deno_core::futures::future::ready(Ok(()))); } let specifier = specifier.clone(); + let graph_container = self.graph_container.clone(); let module_load_preparer = self.shared.module_load_preparer.clone(); let root_permissions = if is_dynamic { @@ -789,9 +576,19 @@ impl ModuleLoader for CliModuleLoader { let lib = self.lib; async move { + let mut update_permit = graph_container.acquire_update_permit().await; + let graph = update_permit.graph_mut(); module_load_preparer - .prepare_module_load(vec![specifier], is_dynamic, lib, root_permissions) - .await + .prepare_module_load( + graph, + &[specifier], + is_dynamic, + lib, + root_permissions, + ) + .await?; + update_permit.commit(); + Ok(()) } .boxed_local() } @@ -824,11 +621,7 @@ impl ModuleLoader for CliModuleLoader { } } -struct CliSourceMapGetter { - shared: Arc, -} - -impl SourceMapGetter for CliSourceMapGetter { +impl SourceMapGetter for CliModuleLoader { fn get_source_map(&self, file_name: &str) -> Option> { let specifier = resolve_url(file_name).ok()?; match specifier.scheme() { @@ -838,7 +631,6 @@ impl SourceMapGetter for CliSourceMapGetter { _ => return None, } let source = self - .shared .prepared_module_loader .load_prepared_module(&specifier, None) .ok()?; @@ -850,7 +642,7 @@ impl SourceMapGetter for CliSourceMapGetter { file_name: &str, line_number: usize, ) -> Option { - let graph = self.shared.graph_container.graph(); + let graph = self.graph_container.graph(); let code = match graph.get(&resolve_url(file_name).ok()?) { Some(deno_graph::Module::Js(module)) => &module.source, Some(deno_graph::Module::Json(module)) => &module.source, From bea3f52afea352cb6c066bf1e61b1a67ce94f8ec Mon Sep 17 00:00:00 2001 From: David Sherret Date: Fri, 3 May 2024 20:05:32 -0400 Subject: [PATCH 4/8] Add ModuleGraphContainer trait to have a separate impl for workers --- cli/workers/mod.rs | 1 + cli/workers/module_graph_container.rs | 98 +++++++++++-- cli/workers/module_loader.rs | 200 ++++++++++++-------------- 3 files changed, 178 insertions(+), 121 deletions(-) diff --git a/cli/workers/mod.rs b/cli/workers/mod.rs index d17f0755b6ca3..b2330c24ec9b4 100644 --- a/cli/workers/mod.rs +++ b/cli/workers/mod.rs @@ -3,4 +3,5 @@ mod module_graph_container; mod module_loader; +pub use module_graph_container::MainModuleGraphContainer; pub use module_loader::CliModuleLoaderFactory; diff --git a/cli/workers/module_graph_container.rs b/cli/workers/module_graph_container.rs index 363c1bd15c782..e3f2ac87bb3a7 100644 --- a/cli/workers/module_graph_container.rs +++ b/cli/workers/module_graph_container.rs @@ -4,16 +4,87 @@ use std::cell::RefCell; use std::rc::Rc; use std::sync::Arc; -use deno_core::unsync::TaskQueue; -use deno_core::unsync::TaskQueuePermit; +use deno_core::parking_lot::RwLock; +use deno_graph::GraphKind; use deno_graph::ModuleGraph; -/// Holds the `ModuleGraph`. +pub trait ModuleGraphContainer: Clone + 'static { + /// Acquires a permit to modify the module graph without other code + /// having the chance to modify it. In the meantime, other code may + /// still read from the existing module graph. + async fn acquire_update_permit(&self) -> impl ModuleGraphUpdatePermit; + /// Gets a copy of the graph. + fn graph(&self) -> Arc; +} + +pub trait ModuleGraphUpdatePermit { + /// Gets the module graph for mutation. + fn graph_mut(&mut self) -> &mut ModuleGraph; + /// Saves the mutated module graph in the container. + fn commit(self); +} + +/// Holds the `ModuleGraph` for the main worker. +#[derive(Clone)] +pub struct MainModuleGraphContainer { + // Allow only one request to update the graph data at a time, + // but allow other requests to read from it at any time even + // while another request is updating the data. + update_queue: Arc, + inner: Arc>>, +} + +impl MainModuleGraphContainer { + pub fn new(graph_kind: GraphKind) -> Self { + Self { + update_queue: Default::default(), + inner: Arc::new(RwLock::new(Arc::new(ModuleGraph::new(graph_kind)))), + } + } +} + +impl ModuleGraphContainer for MainModuleGraphContainer { + async fn acquire_update_permit(&self) -> impl ModuleGraphUpdatePermit { + let permit = self.update_queue.acquire().await; + MainModuleGraphUpdatePermit { + permit, + inner: self.inner.clone(), + graph: (**self.inner.read()).clone(), + } + } + + fn graph(&self) -> Arc { + self.inner.read().clone() + } +} + +/// A permit for updating the module graph. When complete and +/// everything looks fine, calling `.commit()` will store the +/// new graph in the ModuleGraphContainer. +pub struct MainModuleGraphUpdatePermit<'a> { + permit: crate::util::sync::TaskQueuePermit<'a>, + inner: Arc>>, + graph: ModuleGraph, +} + +impl<'a> ModuleGraphUpdatePermit for MainModuleGraphUpdatePermit<'a> { + fn graph_mut(&mut self) -> &mut ModuleGraph { + &mut self.graph + } + + fn commit(self) { + *self.inner.write() = Arc::new(self.graph); + drop(self.permit); // explicit drop for clarity + } +} + +/// Holds the `ModuleGraph` in workers. +#[derive(Clone)] pub struct WorkerModuleGraphContainer { // Allow only one request to update the graph data at a time, // but allow other requests to read from it at any time even // while another request is updating the data. - update_queue: Rc, + update_queue: Rc, inner: Rc>>, } @@ -24,11 +95,10 @@ impl WorkerModuleGraphContainer { inner: Rc::new(RefCell::new(module_graph)), } } +} - /// Acquires a permit to modify the module graph without other code - /// having the chance to modify it. In the meantime, other code may - /// still read from the existing module graph. - pub async fn acquire_update_permit(&self) -> WorkerModuleGraphUpdatePermit { +impl ModuleGraphContainer for WorkerModuleGraphContainer { + async fn acquire_update_permit(&self) -> impl ModuleGraphUpdatePermit { let permit = self.update_queue.acquire().await; WorkerModuleGraphUpdatePermit { permit, @@ -37,7 +107,7 @@ impl WorkerModuleGraphContainer { } } - pub fn graph(&self) -> Arc { + fn graph(&self) -> Arc { self.inner.borrow().clone() } } @@ -46,19 +116,17 @@ impl WorkerModuleGraphContainer { /// everything looks fine, calling `.commit()` will store the /// new graph in the ModuleGraphContainer. pub struct WorkerModuleGraphUpdatePermit { - permit: TaskQueuePermit, + permit: deno_core::unsync::TaskQueuePermit, inner: Rc>>, graph: ModuleGraph, } -impl WorkerModuleGraphUpdatePermit { - /// Gets the module graph for mutation. - pub fn graph_mut(&mut self) -> &mut ModuleGraph { +impl ModuleGraphUpdatePermit for WorkerModuleGraphUpdatePermit { + fn graph_mut(&mut self) -> &mut ModuleGraph { &mut self.graph } - /// Saves the mutated module graph in the container. - pub fn commit(self) { + fn commit(self) { *self.inner.borrow_mut() = Arc::new(self.graph); drop(self.permit); // explicit drop for clarity } diff --git a/cli/workers/module_loader.rs b/cli/workers/module_loader.rs index f90bbf34fd311..a2efefb73b0fb 100644 --- a/cli/workers/module_loader.rs +++ b/cli/workers/module_loader.rs @@ -57,92 +57,10 @@ use std::rc::Rc; use std::str; use std::sync::Arc; +use super::module_graph_container::ModuleGraphContainer; +use super::module_graph_container::ModuleGraphUpdatePermit; use super::module_graph_container::WorkerModuleGraphContainer; -#[derive(Clone)] -struct PreparedModuleLoader { - emitter: Arc, - graph_container: Rc, - parsed_source_cache: Arc, -} - -impl PreparedModuleLoader { - pub fn load_prepared_module( - &self, - specifier: &ModuleSpecifier, - maybe_referrer: Option<&ModuleSpecifier>, - ) -> Result { - if specifier.scheme() == "node" { - unreachable!(); // Node built-in modules should be handled internally. - } - - let graph = self.graph_container.graph(); - match graph.get(specifier) { - Some(deno_graph::Module::Json(JsonModule { - source, - media_type, - specifier, - .. - })) => Ok(ModuleCodeStringSource { - code: source.clone().into(), - found_url: specifier.clone(), - media_type: *media_type, - }), - Some(deno_graph::Module::Js(JsModule { - source, - media_type, - specifier, - .. - })) => { - let code: ModuleCodeString = match media_type { - MediaType::JavaScript - | MediaType::Unknown - | MediaType::Cjs - | MediaType::Mjs - | MediaType::Json => source.clone().into(), - MediaType::Dts | MediaType::Dcts | MediaType::Dmts => { - Default::default() - } - MediaType::TypeScript - | MediaType::Mts - | MediaType::Cts - | MediaType::Jsx - | MediaType::Tsx => { - // get emit text - self - .emitter - .emit_parsed_source(specifier, *media_type, source)? - } - MediaType::TsBuildInfo | MediaType::Wasm | MediaType::SourceMap => { - panic!("Unexpected media type {media_type} for {specifier}") - } - }; - - // at this point, we no longer need the parsed source in memory, so free it - self.parsed_source_cache.free(specifier); - - Ok(ModuleCodeStringSource { - code, - found_url: specifier.clone(), - media_type: *media_type, - }) - } - Some( - deno_graph::Module::External(_) - | deno_graph::Module::Node(_) - | deno_graph::Module::Npm(_), - ) - | None => { - let mut msg = format!("Loading unprepared module: {specifier}"); - if let Some(referrer) = maybe_referrer { - msg = format!("{}, imported from: {}", msg, referrer.as_str()); - } - Err(anyhow!(msg)) - } - } - } -} - struct SharedCliModuleLoaderState { graph_kind: GraphKind, lib_window: TsTypeLib, @@ -205,18 +123,13 @@ impl CliModuleLoaderFactory { root_permissions: PermissionsContainer, dynamic_permissions: PermissionsContainer, ) -> ModuleLoaderAndSourceMapGetter { - let graph_container = - Rc::new(WorkerModuleGraphContainer::new(module_graph)); let loader = Rc::new(CliModuleLoader { lib, root_permissions, dynamic_permissions, - graph_container: graph_container.clone(), - prepared_module_loader: PreparedModuleLoader { - emitter: self.shared.emitter.clone(), - graph_container, - parsed_source_cache: self.shared.parsed_source_cache.clone(), - }, + graph_container: WorkerModuleGraphContainer::new(module_graph), + emitter: self.shared.emitter.clone(), + parsed_source_cache: self.shared.parsed_source_cache.clone(), shared: self.shared.clone(), }); ModuleLoaderAndSourceMapGetter { @@ -256,7 +169,7 @@ impl ModuleLoaderFactory for CliModuleLoaderFactory { } } -struct CliModuleLoader { +struct CliModuleLoader { lib: TsTypeLib, /// The initial set of permissions used to resolve the static imports in the /// worker. These are "allow all" for main worker, and parent thread @@ -266,11 +179,12 @@ struct CliModuleLoader { /// "root permissions" for Web Worker. dynamic_permissions: PermissionsContainer, shared: Arc, - graph_container: Rc, - prepared_module_loader: PreparedModuleLoader, + emitter: Arc, + parsed_source_cache: Arc, + graph_container: TGraphContainer, } -impl CliModuleLoader { +impl CliModuleLoader { fn load_sync( &self, specifier: &ModuleSpecifier, @@ -290,9 +204,7 @@ impl CliModuleLoader { { result? } else { - self - .prepared_module_loader - .load_prepared_module(specifier, maybe_referrer)? + self.load_prepared_module(specifier, maybe_referrer)? }; let code = if self.shared.is_inspecting { // we need the code with the source map in order for @@ -508,9 +420,86 @@ impl CliModuleLoader { .map(|timestamp| timestamp.to_string())?; Ok(Some(timestamp)) } + + fn load_prepared_module( + &self, + specifier: &ModuleSpecifier, + maybe_referrer: Option<&ModuleSpecifier>, + ) -> Result { + if specifier.scheme() == "node" { + unreachable!(); // Node built-in modules should be handled internally. + } + + let graph = self.graph_container.graph(); + match graph.get(specifier) { + Some(deno_graph::Module::Json(JsonModule { + source, + media_type, + specifier, + .. + })) => Ok(ModuleCodeStringSource { + code: source.clone().into(), + found_url: specifier.clone(), + media_type: *media_type, + }), + Some(deno_graph::Module::Js(JsModule { + source, + media_type, + specifier, + .. + })) => { + let code: ModuleCodeString = match media_type { + MediaType::JavaScript + | MediaType::Unknown + | MediaType::Cjs + | MediaType::Mjs + | MediaType::Json => source.clone().into(), + MediaType::Dts | MediaType::Dcts | MediaType::Dmts => { + Default::default() + } + MediaType::TypeScript + | MediaType::Mts + | MediaType::Cts + | MediaType::Jsx + | MediaType::Tsx => { + // get emit text + self + .emitter + .emit_parsed_source(specifier, *media_type, source)? + } + MediaType::TsBuildInfo | MediaType::Wasm | MediaType::SourceMap => { + panic!("Unexpected media type {media_type} for {specifier}") + } + }; + + // at this point, we no longer need the parsed source in memory, so free it + self.parsed_source_cache.free(specifier); + + Ok(ModuleCodeStringSource { + code, + found_url: specifier.clone(), + media_type: *media_type, + }) + } + Some( + deno_graph::Module::External(_) + | deno_graph::Module::Node(_) + | deno_graph::Module::Npm(_), + ) + | None => { + let mut msg = format!("Loading unprepared module: {specifier}"); + if let Some(referrer) = maybe_referrer { + msg = format!("{}, imported from: {}", msg, referrer.as_str()); + } + Err(anyhow!(msg)) + } + } + } } -impl ModuleLoader for CliModuleLoader { +impl ModuleLoader + for CliModuleLoader +{ fn resolve( &self, specifier: &str, @@ -617,11 +606,13 @@ impl ModuleLoader for CliModuleLoader { ); } } - async {}.boxed_local() + std::future::ready(()).boxed_local() } } -impl SourceMapGetter for CliModuleLoader { +impl SourceMapGetter + for CliModuleLoader +{ fn get_source_map(&self, file_name: &str) -> Option> { let specifier = resolve_url(file_name).ok()?; match specifier.scheme() { @@ -630,10 +621,7 @@ impl SourceMapGetter for CliModuleLoader { "wasm" | "file" | "http" | "https" | "data" | "blob" => (), _ => return None, } - let source = self - .prepared_module_loader - .load_prepared_module(&specifier, None) - .ok()?; + let source = self.load_prepared_module(&specifier, None).ok()?; source_map_from_code(&source.code) } From 99f4ad52a3f6f9756103e377abc873cc57a5c6a5 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Fri, 3 May 2024 20:46:53 -0400 Subject: [PATCH 5/8] Better. --- cli/factory.rs | 32 ++- ..._graph_container.rs => graph_container.rs} | 142 ++++++----- cli/lsp/testing/execution.rs | 8 +- cli/main.rs | 19 +- cli/module_load_preparer.rs | 206 ---------------- cli/{workers => }/module_loader.rs | 230 +++++++++++++++--- cli/standalone/mod.rs | 4 - cli/tools/bench/mod.rs | 23 +- cli/tools/installer.rs | 6 +- cli/tools/jupyter/mod.rs | 4 - cli/tools/repl/mod.rs | 2 - cli/tools/run/mod.rs | 30 +-- cli/tools/test/mod.rs | 29 +-- cli/worker.rs | 13 +- cli/workers/mod.rs | 7 - 15 files changed, 337 insertions(+), 418 deletions(-) rename cli/{workers/module_graph_container.rs => graph_container.rs} (51%) delete mode 100644 cli/module_load_preparer.rs rename cli/{workers => }/module_loader.rs (78%) delete mode 100644 cli/workers/mod.rs diff --git a/cli/factory.rs b/cli/factory.rs index ff56e453b5b91..51fb865d318bf 100644 --- a/cli/factory.rs +++ b/cli/factory.rs @@ -21,12 +21,13 @@ use crate::cache::NodeAnalysisCache; use crate::cache::ParsedSourceCache; use crate::emit::Emitter; use crate::file_fetcher::FileFetcher; +use crate::graph_container::MainModuleGraphContainer; use crate::graph_util::FileWatcherReporter; use crate::graph_util::ModuleGraphBuilder; use crate::graph_util::ModuleGraphCreator; use crate::http_util::HttpClient; -use crate::module_load_preparer::MainModuleGraphPreparer; -use crate::module_load_preparer::ModuleLoadPreparer; +use crate::module_loader::CliModuleLoaderFactory; +use crate::module_loader::ModuleLoadPreparer; use crate::node::CliCjsCodeAnalyzer; use crate::node::CliNodeCodeTranslator; use crate::npm::create_cli_npm_resolver; @@ -52,7 +53,6 @@ use crate::util::progress_bar::ProgressBar; use crate::util::progress_bar::ProgressBarStyle; use crate::worker::CliMainWorkerFactory; use crate::worker::CliMainWorkerOptions; -use crate::workers::CliModuleLoaderFactory; use std::path::PathBuf; use deno_core::error::AnyError; @@ -156,6 +156,7 @@ struct CliFactoryServices { emit_cache: Deferred, emitter: Deferred>, fs: Deferred>, + main_graph_container: Deferred>, lockfile: Deferred>>>, maybe_import_map: Deferred>>, maybe_inspector_server: Deferred>>, @@ -670,6 +671,21 @@ impl CliFactory { .await } + pub async fn main_module_graph_container( + &self, + ) -> Result<&Arc, AnyError> { + self + .services + .main_graph_container + .get_or_try_init_async(async { + Ok(Arc::new(MainModuleGraphContainer::new( + self.cli_options().clone(), + self.module_load_preparer().await?.clone(), + ))) + }) + .await + } + pub fn maybe_inspector_server( &self, ) -> Result<&Option>, AnyError> { @@ -753,15 +769,6 @@ impl CliFactory { )) } - pub async fn create_main_module_graph_preparer( - &self, - ) -> Result { - Ok(MainModuleGraphPreparer::new( - self.options.clone(), - self.module_load_preparer().await?.clone(), - )) - } - pub async fn create_cli_main_worker_factory( &self, ) -> Result { @@ -789,6 +796,7 @@ impl CliFactory { None }, self.emitter()?.clone(), + self.main_module_graph_container().await?.clone(), self.module_info_cache()?.clone(), self.module_load_preparer().await?.clone(), cli_node_resolver.clone(), diff --git a/cli/workers/module_graph_container.rs b/cli/graph_container.rs similarity index 51% rename from cli/workers/module_graph_container.rs rename to cli/graph_container.rs index e3f2ac87bb3a7..ec18ffaab45d4 100644 --- a/cli/workers/module_graph_container.rs +++ b/cli/graph_container.rs @@ -1,12 +1,17 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -use std::cell::RefCell; -use std::rc::Rc; use std::sync::Arc; +use deno_ast::ModuleSpecifier; +use deno_core::error::AnyError; use deno_core::parking_lot::RwLock; -use deno_graph::GraphKind; +use deno_core::resolve_url_or_path; use deno_graph::ModuleGraph; +use deno_runtime::colors; +use deno_runtime::permissions::PermissionsContainer; + +use crate::args::CliOptions; +use crate::module_loader::ModuleLoadPreparer; pub trait ModuleGraphContainer: Clone + 'static { /// Acquires a permit to modify the module graph without other code @@ -17,6 +22,9 @@ pub trait ModuleGraphContainer: Clone + 'static { fn graph(&self) -> Arc; } +/// A permit for updating the module graph. When complete and +/// everything looks fine, calling `.commit()` will store the +/// new graph in the ModuleGraphContainer. pub trait ModuleGraphUpdatePermit { /// Gets the module graph for mutation. fn graph_mut(&mut self) -> &mut ModuleGraph; @@ -32,15 +40,85 @@ pub struct MainModuleGraphContainer { // while another request is updating the data. update_queue: Arc, inner: Arc>>, + cli_options: Arc, + module_load_preparer: Arc, } impl MainModuleGraphContainer { - pub fn new(graph_kind: GraphKind) -> Self { + pub fn new( + cli_options: Arc, + module_load_preparer: Arc, + ) -> Self { Self { update_queue: Default::default(), - inner: Arc::new(RwLock::new(Arc::new(ModuleGraph::new(graph_kind)))), + inner: Arc::new(RwLock::new(Arc::new(ModuleGraph::new( + cli_options.graph_kind(), + )))), + cli_options, + module_load_preparer, } } + + pub async fn check_specifiers( + &self, + specifiers: &[ModuleSpecifier], + ) -> Result<(), AnyError> { + let mut graph_permit = self.acquire_update_permit().await; + let graph = graph_permit.graph_mut(); + self + .module_load_preparer + .prepare_module_load( + graph, + specifiers, + false, + self.cli_options.ts_type_lib_window(), + PermissionsContainer::allow_all(), + ) + .await?; + graph_permit.commit(); + Ok(()) + } + + /// Helper around prepare_module_load that loads and type checks + /// the provided files. + pub async fn load_and_type_check_files( + &self, + files: &[String], + ) -> Result<(), AnyError> { + let specifiers = self.collect_specifiers(files)?; + + if specifiers.is_empty() { + log::warn!("{} No matching files found.", colors::yellow("Warning")); + } + + self.check_specifiers(&specifiers).await + } + + pub fn collect_specifiers( + &self, + files: &[String], + ) -> Result, AnyError> { + let excludes = self.cli_options.resolve_config_excludes()?; + Ok( + files + .iter() + .filter_map(|file| { + let file_url = + resolve_url_or_path(file, self.cli_options.initial_cwd()).ok()?; + if file_url.scheme() != "file" { + return Some(file_url); + } + // ignore local files that match any of files listed in `exclude` option + let file_path = file_url.to_file_path().ok()?; + if excludes.matches_path(&file_path) { + None + } else { + Some(file_url) + } + }) + .collect::>(), + ) + } } impl ModuleGraphContainer for MainModuleGraphContainer { @@ -77,57 +155,3 @@ impl<'a> ModuleGraphUpdatePermit for MainModuleGraphUpdatePermit<'a> { drop(self.permit); // explicit drop for clarity } } - -/// Holds the `ModuleGraph` in workers. -#[derive(Clone)] -pub struct WorkerModuleGraphContainer { - // Allow only one request to update the graph data at a time, - // but allow other requests to read from it at any time even - // while another request is updating the data. - update_queue: Rc, - inner: Rc>>, -} - -impl WorkerModuleGraphContainer { - pub fn new(module_graph: Arc) -> Self { - Self { - update_queue: Default::default(), - inner: Rc::new(RefCell::new(module_graph)), - } - } -} - -impl ModuleGraphContainer for WorkerModuleGraphContainer { - async fn acquire_update_permit(&self) -> impl ModuleGraphUpdatePermit { - let permit = self.update_queue.acquire().await; - WorkerModuleGraphUpdatePermit { - permit, - inner: self.inner.clone(), - graph: (**self.inner.borrow()).clone(), - } - } - - fn graph(&self) -> Arc { - self.inner.borrow().clone() - } -} - -/// A permit for updating the module graph. When complete and -/// everything looks fine, calling `.commit()` will store the -/// new graph in the ModuleGraphContainer. -pub struct WorkerModuleGraphUpdatePermit { - permit: deno_core::unsync::TaskQueuePermit, - inner: Rc>>, - graph: ModuleGraph, -} - -impl ModuleGraphUpdatePermit for WorkerModuleGraphUpdatePermit { - fn graph_mut(&mut self) -> &mut ModuleGraph { - &mut self.graph - } - - fn commit(self) { - *self.inner.borrow_mut() = Arc::new(self.graph); - drop(self.permit); // explicit drop for clarity - } -} diff --git a/cli/lsp/testing/execution.rs b/cli/lsp/testing/execution.rs index 8252e15728c8c..9c611e0fa4ed0 100644 --- a/cli/lsp/testing/execution.rs +++ b/cli/lsp/testing/execution.rs @@ -219,11 +219,10 @@ impl TestRun { // file would have impact on other files, which is undesirable. let permissions = Permissions::from_options(&factory.cli_options().permissions_options())?; - let mut main_graph_preparer = - factory.create_main_module_graph_preparer().await?; + let main_graph_container = factory.main_module_graph_container().await?; test::check_specifiers( factory.file_fetcher()?, - &mut main_graph_preparer, + main_graph_container, self .queue .iter() @@ -264,7 +263,6 @@ impl TestRun { let mut test_steps = IndexMap::new(); let worker_factory = Arc::new(factory.create_cli_main_worker_factory().await?); - let module_graph = Arc::new(main_graph_preparer.into_graph()); let join_handles = queue.into_iter().map(move |specifier| { let specifier = specifier.clone(); @@ -286,7 +284,6 @@ impl TestRun { .unwrap_or_default(), }; let token = self.token.clone(); - let module_graph = module_graph.clone(); spawn_blocking(move || { if fail_fast_tracker.should_stop() { @@ -301,7 +298,6 @@ impl TestRun { worker_factory, permissions, specifier, - module_graph, worker_sender, fail_fast_tracker, test::TestSpecifierOptions { diff --git a/cli/main.rs b/cli/main.rs index 60c917fd8ea1d..aa99fa9916e11 100644 --- a/cli/main.rs +++ b/cli/main.rs @@ -8,12 +8,13 @@ mod emit; mod errors; mod factory; mod file_fetcher; +mod graph_container; mod graph_util; mod http_util; mod js; mod jsr; mod lsp; -mod module_load_preparer; +mod module_loader; mod napi; mod node; mod npm; @@ -25,12 +26,12 @@ mod tsc; mod util; mod version; mod worker; -mod workers; use crate::args::flags_from_vec; use crate::args::DenoSubcommand; use crate::args::Flags; use crate::args::DENO_FUTURE; +use crate::graph_container::ModuleGraphContainer; use crate::util::display; use crate::util::v8::get_v8_flags_from_env; use crate::util::v8::init_v8_flags; @@ -114,18 +115,18 @@ async fn run_subcommand(flags: Flags) -> Result { DenoSubcommand::Cache(cache_flags) => spawn_subcommand(async move { let factory = CliFactory::from_flags(flags)?; let emitter = factory.emitter()?; - let mut module_graph_preparer = - factory.create_main_module_graph_preparer().await?; - module_graph_preparer + let main_graph_container = + factory.main_module_graph_container().await?; + main_graph_container .load_and_type_check_files(&cache_flags.files) .await?; - emitter.cache_module_emits(&module_graph_preparer.graph()) + emitter.cache_module_emits(&main_graph_container.graph()) }), DenoSubcommand::Check(check_flags) => spawn_subcommand(async move { let factory = CliFactory::from_flags(flags)?; - let mut module_graph_preparer = - factory.create_main_module_graph_preparer().await?; - module_graph_preparer + let main_graph_container = + factory.main_module_graph_container().await?; + main_graph_container .load_and_type_check_files(&check_flags.files) .await }), diff --git a/cli/module_load_preparer.rs b/cli/module_load_preparer.rs deleted file mode 100644 index 4016cc5edb89c..0000000000000 --- a/cli/module_load_preparer.rs +++ /dev/null @@ -1,206 +0,0 @@ -// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. - -use crate::args::CliOptions; -use crate::args::TsTypeLib; -use crate::graph_util::graph_lock_or_exit; -use crate::graph_util::CreateGraphOptions; -use crate::graph_util::ModuleGraphBuilder; -use crate::tools::check; -use crate::tools::check::TypeChecker; -use crate::util::progress_bar::ProgressBar; - -use deno_core::anyhow::Context; -use deno_core::error::AnyError; -use deno_core::parking_lot::Mutex; -use deno_core::resolve_url_or_path; -use deno_core::ModuleSpecifier; -use deno_graph::ModuleGraph; -use deno_lockfile::Lockfile; -use deno_runtime::permissions::PermissionsContainer; -use deno_terminal::colors; -use std::sync::Arc; - -pub struct ModuleLoadPreparer { - options: Arc, - lockfile: Option>>, - module_graph_builder: Arc, - progress_bar: ProgressBar, - type_checker: Arc, -} - -impl ModuleLoadPreparer { - #[allow(clippy::too_many_arguments)] - pub fn new( - options: Arc, - lockfile: Option>>, - module_graph_builder: Arc, - progress_bar: ProgressBar, - type_checker: Arc, - ) -> Self { - Self { - options, - lockfile, - module_graph_builder, - progress_bar, - type_checker, - } - } - - /// This method must be called for a module or a static importer of that - /// module before attempting to `load()` it from a `JsRuntime`. It will - /// populate the graph data in memory with the necessary source code, write - /// emits where necessary or report any module graph / type checking errors. - #[allow(clippy::too_many_arguments)] - pub async fn prepare_module_load( - &self, - graph: &mut ModuleGraph, - roots: &[ModuleSpecifier], - is_dynamic: bool, - lib: TsTypeLib, - permissions: PermissionsContainer, - ) -> Result<(), AnyError> { - log::debug!("Preparing module load."); - let _pb_clear_guard = self.progress_bar.clear_guard(); - - let mut cache = self.module_graph_builder.create_fetch_cacher(permissions); - log::debug!("Building module graph."); - let has_type_checked = !graph.roots.is_empty(); - - self - .module_graph_builder - .build_graph_with_npm_resolution( - graph, - CreateGraphOptions { - is_dynamic, - graph_kind: graph.graph_kind(), - roots: roots.to_vec(), - loader: Some(&mut cache), - }, - ) - .await?; - - self.module_graph_builder.graph_roots_valid(graph, &roots)?; - - // If there is a lockfile... - if let Some(lockfile) = &self.lockfile { - let mut lockfile = lockfile.lock(); - // validate the integrity of all the modules - graph_lock_or_exit(graph, &mut lockfile); - // update it with anything new - lockfile.write().context("Failed writing lockfile.")?; - } - - drop(_pb_clear_guard); - - // type check if necessary - if self.options.type_check_mode().is_true() && !has_type_checked { - self - .type_checker - .check( - // todo(perf): since this is only done the first time the graph is - // created, we could avoid the clone of the graph here by providing - // the actual graph on the first run and then getting the Arc - // back from the return value. - graph.clone(), - check::CheckOptions { - build_fast_check_graph: true, - lib, - log_ignored_options: false, - reload: self.options.reload_flag(), - type_check_mode: self.options.type_check_mode(), - }, - ) - .await?; - } - - log::debug!("Prepared module load."); - - Ok(()) - } -} - -pub struct MainModuleGraphPreparer { - options: Arc, - module_load_preparer: Arc, - graph: ModuleGraph, -} - -impl MainModuleGraphPreparer { - pub fn new( - options: Arc, - module_load_preparer: Arc, - ) -> Self { - Self { - graph: ModuleGraph::new(options.graph_kind()), - options, - module_load_preparer, - } - } - - pub fn into_graph(self) -> ModuleGraph { - self.graph - } - - pub fn graph(&self) -> &ModuleGraph { - &self.graph - } - - pub async fn check_specifiers( - &mut self, - specifiers: &[ModuleSpecifier], - ) -> Result<(), AnyError> { - let lib = self.options.ts_type_lib_window(); - self - .module_load_preparer - .prepare_module_load( - &mut self.graph, - specifiers, - false, - lib, - PermissionsContainer::allow_all(), - ) - .await?; - Ok(()) - } - - /// Helper around prepare_module_load that loads and type checks - /// the provided files. - pub async fn load_and_type_check_files( - &mut self, - files: &[String], - ) -> Result<(), AnyError> { - let specifiers = self.collect_specifiers(files)?; - - if specifiers.is_empty() { - log::warn!("{} No matching files found.", colors::yellow("Warning")); - } - - self.check_specifiers(&specifiers).await - } - - fn collect_specifiers( - &self, - files: &[String], - ) -> Result, AnyError> { - let excludes = self.options.resolve_config_excludes()?; - Ok( - files - .iter() - .filter_map(|file| { - let file_url = - resolve_url_or_path(file, self.options.initial_cwd()).ok()?; - if file_url.scheme() != "file" { - return Some(file_url); - } - // ignore local files that match any of files listed in `exclude` option - let file_path = file_url.to_file_path().ok()?; - if excludes.matches_path(&file_path) { - None - } else { - Some(file_url) - } - }) - .collect::>(), - ) - } -} diff --git a/cli/workers/module_loader.rs b/cli/module_loader.rs similarity index 78% rename from cli/workers/module_loader.rs rename to cli/module_loader.rs index a2efefb73b0fb..852abf07d85e8 100644 --- a/cli/workers/module_loader.rs +++ b/cli/module_loader.rs @@ -1,23 +1,11 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -use crate::args::jsr_url; -use crate::args::CliOptions; -use crate::args::DenoSubcommand; -use crate::args::TsTypeLib; -use crate::cache::CodeCache; -use crate::cache::ModuleInfoCache; -use crate::cache::ParsedSourceCache; -use crate::emit::Emitter; -use crate::module_load_preparer::ModuleLoadPreparer; -use crate::node; -use crate::resolver::CliGraphResolver; -use crate::resolver::CliNodeResolver; -use crate::resolver::ModuleCodeStringSource; -use crate::resolver::NpmModuleLoader; -use crate::util::text_encoding::code_without_source_map; -use crate::util::text_encoding::source_map_from_code; -use crate::worker::ModuleLoaderAndSourceMapGetter; -use crate::worker::ModuleLoaderFactory; +use std::borrow::Cow; +use std::cell::RefCell; +use std::pin::Pin; +use std::rc::Rc; +use std::str; +use std::sync::Arc; use deno_ast::MediaType; use deno_core::anyhow::anyhow; @@ -28,6 +16,7 @@ use deno_core::error::generic_error; use deno_core::error::AnyError; use deno_core::futures::future::FutureExt; use deno_core::futures::Future; +use deno_core::parking_lot::Mutex; use deno_core::resolve_url; use deno_core::ModuleCodeString; use deno_core::ModuleLoader; @@ -46,20 +35,138 @@ use deno_graph::JsonModule; use deno_graph::Module; use deno_graph::ModuleGraph; use deno_graph::Resolution; +use deno_lockfile::Lockfile; use deno_runtime::code_cache; use deno_runtime::deno_node::NodeResolutionMode; use deno_runtime::fs_util::code_timestamp; use deno_runtime::permissions::PermissionsContainer; use deno_semver::npm::NpmPackageReqReference; -use std::borrow::Cow; -use std::pin::Pin; -use std::rc::Rc; -use std::str; -use std::sync::Arc; -use super::module_graph_container::ModuleGraphContainer; -use super::module_graph_container::ModuleGraphUpdatePermit; -use super::module_graph_container::WorkerModuleGraphContainer; +use crate::args::jsr_url; +use crate::args::CliOptions; +use crate::args::DenoSubcommand; +use crate::args::TsTypeLib; +use crate::cache::CodeCache; +use crate::cache::ModuleInfoCache; +use crate::cache::ParsedSourceCache; +use crate::emit::Emitter; +use crate::graph_container::MainModuleGraphContainer; +use crate::graph_container::ModuleGraphContainer; +use crate::graph_container::ModuleGraphUpdatePermit; +use crate::graph_util::graph_lock_or_exit; +use crate::graph_util::CreateGraphOptions; +use crate::graph_util::ModuleGraphBuilder; +use crate::node; +use crate::resolver::CliGraphResolver; +use crate::resolver::CliNodeResolver; +use crate::resolver::ModuleCodeStringSource; +use crate::resolver::NpmModuleLoader; +use crate::tools::check; +use crate::tools::check::TypeChecker; +use crate::util::progress_bar::ProgressBar; +use crate::util::text_encoding::code_without_source_map; +use crate::util::text_encoding::source_map_from_code; +use crate::worker::ModuleLoaderAndSourceMapGetter; +use crate::worker::ModuleLoaderFactory; + +pub struct ModuleLoadPreparer { + options: Arc, + lockfile: Option>>, + module_graph_builder: Arc, + progress_bar: ProgressBar, + type_checker: Arc, +} + +impl ModuleLoadPreparer { + #[allow(clippy::too_many_arguments)] + pub fn new( + options: Arc, + lockfile: Option>>, + module_graph_builder: Arc, + progress_bar: ProgressBar, + type_checker: Arc, + ) -> Self { + Self { + options, + lockfile, + module_graph_builder, + progress_bar, + type_checker, + } + } + + /// This method must be called for a module or a static importer of that + /// module before attempting to `load()` it from a `JsRuntime`. It will + /// populate the graph data in memory with the necessary source code, write + /// emits where necessary or report any module graph / type checking errors. + #[allow(clippy::too_many_arguments)] + pub async fn prepare_module_load( + &self, + graph: &mut ModuleGraph, + roots: &[ModuleSpecifier], + is_dynamic: bool, + lib: TsTypeLib, + permissions: PermissionsContainer, + ) -> Result<(), AnyError> { + log::debug!("Preparing module load."); + let _pb_clear_guard = self.progress_bar.clear_guard(); + + let mut cache = self.module_graph_builder.create_fetch_cacher(permissions); + log::debug!("Building module graph."); + let has_type_checked = !graph.roots.is_empty(); + + self + .module_graph_builder + .build_graph_with_npm_resolution( + graph, + CreateGraphOptions { + is_dynamic, + graph_kind: graph.graph_kind(), + roots: roots.to_vec(), + loader: Some(&mut cache), + }, + ) + .await?; + + self.module_graph_builder.graph_roots_valid(graph, roots)?; + + // If there is a lockfile... + if let Some(lockfile) = &self.lockfile { + let mut lockfile = lockfile.lock(); + // validate the integrity of all the modules + graph_lock_or_exit(graph, &mut lockfile); + // update it with anything new + lockfile.write().context("Failed writing lockfile.")?; + } + + drop(_pb_clear_guard); + + // type check if necessary + if self.options.type_check_mode().is_true() && !has_type_checked { + self + .type_checker + .check( + // todo(perf): since this is only done the first time the graph is + // created, we could avoid the clone of the graph here by providing + // the actual graph on the first run and then getting the Arc + // back from the return value. + graph.clone(), + check::CheckOptions { + build_fast_check_graph: true, + lib, + log_ignored_options: false, + reload: self.options.reload_flag(), + type_check_mode: self.options.type_check_mode(), + }, + ) + .await?; + } + + log::debug!("Prepared module load."); + + Ok(()) + } +} struct SharedCliModuleLoaderState { graph_kind: GraphKind, @@ -69,12 +176,13 @@ struct SharedCliModuleLoaderState { is_repl: bool, code_cache: Option>, emitter: Arc, - resolver: Arc, + main_module_graph_container: Arc, module_info_cache: Arc, module_load_preparer: Arc, node_resolver: Arc, npm_module_loader: NpmModuleLoader, parsed_source_cache: Arc, + resolver: Arc, } pub struct CliModuleLoaderFactory { @@ -87,6 +195,7 @@ impl CliModuleLoaderFactory { options: &CliOptions, code_cache: Option>, emitter: Arc, + main_module_graph_container: Arc, module_info_cache: Arc, module_load_preparer: Arc, node_resolver: Arc, @@ -106,6 +215,7 @@ impl CliModuleLoaderFactory { ), code_cache, emitter, + main_module_graph_container, module_info_cache, module_load_preparer, node_resolver, @@ -116,9 +226,9 @@ impl CliModuleLoaderFactory { } } - fn create_with_lib( + fn create_with_lib( &self, - module_graph: Arc, + graph_container: TGraphContainer, lib: TsTypeLib, root_permissions: PermissionsContainer, dynamic_permissions: PermissionsContainer, @@ -127,7 +237,7 @@ impl CliModuleLoaderFactory { lib, root_permissions, dynamic_permissions, - graph_container: WorkerModuleGraphContainer::new(module_graph), + graph_container, emitter: self.shared.emitter.clone(), parsed_source_cache: self.shared.parsed_source_cache.clone(), shared: self.shared.clone(), @@ -142,12 +252,11 @@ impl CliModuleLoaderFactory { impl ModuleLoaderFactory for CliModuleLoaderFactory { fn create_for_main( &self, - starting_module_graph: Arc, root_permissions: PermissionsContainer, dynamic_permissions: PermissionsContainer, ) -> ModuleLoaderAndSourceMapGetter { self.create_with_lib( - starting_module_graph, + (*self.shared.main_module_graph_container).clone(), self.shared.lib_window, root_permissions, dynamic_permissions, @@ -161,7 +270,9 @@ impl ModuleLoaderFactory for CliModuleLoaderFactory { ) -> ModuleLoaderAndSourceMapGetter { self.create_with_lib( // create a fresh module graph for the worker - Arc::new(ModuleGraph::new(self.shared.graph_kind)), + WorkerModuleGraphContainer::new(Arc::new(ModuleGraph::new( + self.shared.graph_kind, + ))), self.shared.lib_worker, root_permissions, dynamic_permissions, @@ -549,7 +660,7 @@ impl ModuleLoader _maybe_referrer: Option, is_dynamic: bool, ) -> Pin>>> { - if self.shared.node_resolver.in_npm_package(&specifier) { + if self.shared.node_resolver.in_npm_package(specifier) { return Box::pin(deno_core::futures::future::ready(Ok(()))); } @@ -649,3 +760,54 @@ impl SourceMapGetter } } } + +/// Holds the `ModuleGraph` in workers. +#[derive(Clone)] +struct WorkerModuleGraphContainer { + // Allow only one request to update the graph data at a time, + // but allow other requests to read from it at any time even + // while another request is updating the data. + update_queue: Rc, + inner: Rc>>, +} + +impl WorkerModuleGraphContainer { + pub fn new(module_graph: Arc) -> Self { + Self { + update_queue: Default::default(), + inner: Rc::new(RefCell::new(module_graph)), + } + } +} + +impl ModuleGraphContainer for WorkerModuleGraphContainer { + async fn acquire_update_permit(&self) -> impl ModuleGraphUpdatePermit { + let permit = self.update_queue.acquire().await; + WorkerModuleGraphUpdatePermit { + permit, + inner: self.inner.clone(), + graph: (**self.inner.borrow()).clone(), + } + } + + fn graph(&self) -> Arc { + self.inner.borrow().clone() + } +} + +struct WorkerModuleGraphUpdatePermit { + permit: deno_core::unsync::TaskQueuePermit, + inner: Rc>>, + graph: ModuleGraph, +} + +impl ModuleGraphUpdatePermit for WorkerModuleGraphUpdatePermit { + fn graph_mut(&mut self) -> &mut ModuleGraph { + &mut self.graph + } + + fn commit(self) { + *self.inner.borrow_mut() = Arc::new(self.graph); + drop(self.permit); // explicit drop for clarity + } +} diff --git a/cli/standalone/mod.rs b/cli/standalone/mod.rs index 0eb34c1d8e06c..543b680861420 100644 --- a/cli/standalone/mod.rs +++ b/cli/standalone/mod.rs @@ -43,7 +43,6 @@ use deno_core::ModuleSpecifier; use deno_core::ModuleType; use deno_core::RequestedModuleType; use deno_core::ResolutionKind; -use deno_graph::ModuleGraph; use deno_runtime::deno_fs; use deno_runtime::deno_node::analyze::NodeCodeTranslator; use deno_runtime::deno_node::NodeResolutionMode; @@ -282,7 +281,6 @@ struct StandaloneModuleLoaderFactory { impl ModuleLoaderFactory for StandaloneModuleLoaderFactory { fn create_for_main( &self, - _module_graph: Arc, root_permissions: PermissionsContainer, dynamic_permissions: PermissionsContainer, ) -> ModuleLoaderAndSourceMapGetter { @@ -597,8 +595,6 @@ pub async fn run( .create_main_worker( WorkerExecutionMode::Run, main_module.clone(), - // todo(THIS PR): remove this - Arc::new(ModuleGraph::new(deno_graph::GraphKind::All)), permissions, ) .await?; diff --git a/cli/tools/bench/mod.rs b/cli/tools/bench/mod.rs index 0214c9db885f7..bb6b86772fec2 100644 --- a/cli/tools/bench/mod.rs +++ b/cli/tools/bench/mod.rs @@ -31,7 +31,6 @@ use deno_core::unsync::spawn_blocking; use deno_core::v8; use deno_core::ModuleSpecifier; use deno_core::PollEventLoopOptions; -use deno_graph::ModuleGraph; use deno_runtime::permissions::Permissions; use deno_runtime::permissions::PermissionsContainer; use deno_runtime::tokio_util::create_and_run_current_thread; @@ -150,7 +149,6 @@ async fn bench_specifier( worker_factory: Arc, permissions: Permissions, specifier: ModuleSpecifier, - main_module_graph: Arc, sender: UnboundedSender, filter: TestFilter, ) -> Result<(), AnyError> { @@ -158,7 +156,6 @@ async fn bench_specifier( worker_factory, permissions, specifier.clone(), - main_module_graph, &sender, filter, ) @@ -184,7 +181,6 @@ async fn bench_specifier_inner( worker_factory: Arc, permissions: Permissions, specifier: ModuleSpecifier, - main_module_graph: Arc, sender: &UnboundedSender, filter: TestFilter, ) -> Result<(), AnyError> { @@ -192,7 +188,6 @@ async fn bench_specifier_inner( .create_custom_worker( WorkerExecutionMode::Bench, specifier.clone(), - main_module_graph, PermissionsContainer::new(permissions), vec![ops::bench::deno_bench::init_ops(sender.clone())], Default::default(), @@ -273,7 +268,6 @@ async fn bench_specifiers( worker_factory: Arc, permissions: &Permissions, specifiers: Vec, - main_module_graph: Arc, options: BenchSpecifierOptions, ) -> Result<(), AnyError> { let (sender, mut receiver) = unbounded_channel::(); @@ -285,13 +279,11 @@ async fn bench_specifiers( let permissions = permissions.clone(); let sender = sender.clone(); let options = option_for_handles.clone(); - let main_module_graph = main_module_graph.clone(); spawn_blocking(move || { let future = bench_specifier( worker_factory, permissions, specifier, - main_module_graph, sender, options.filter, ); @@ -434,9 +426,8 @@ pub async fn run_benchmarks( return Err(generic_error("No bench modules found")); } - let mut main_graph_preparer = - factory.create_main_module_graph_preparer().await?; - main_graph_preparer.check_specifiers(&specifiers).await?; + let main_graph_container = factory.main_module_graph_container().await?; + main_graph_container.check_specifiers(&specifiers).await?; if bench_options.no_run { return Ok(()); @@ -449,7 +440,6 @@ pub async fn run_benchmarks( worker_factory, &permissions, specifiers, - Arc::new(main_graph_preparer.into_graph()), BenchSpecifierOptions { filter: TestFilter::from_flag(&bench_options.filter), json: bench_options.json, @@ -545,9 +535,11 @@ pub async fn run_benchmarks_with_watch( .filter(|specifier| bench_modules_to_reload.contains(specifier)) .collect::>(); - let mut main_graph_preparer = - factory.create_main_module_graph_preparer().await?; - main_graph_preparer.check_specifiers(&specifiers).await?; + factory + .main_module_graph_container() + .await? + .check_specifiers(&specifiers) + .await?; if bench_options.no_run { return Ok(()); @@ -558,7 +550,6 @@ pub async fn run_benchmarks_with_watch( worker_factory, &permissions, specifiers, - Arc::new(main_graph_preparer.into_graph()), BenchSpecifierOptions { filter: TestFilter::from_flag(&bench_options.filter), json: bench_options.json, diff --git a/cli/tools/installer.rs b/cli/tools/installer.rs index 515b15f2156de..e07ed5d5fb16d 100644 --- a/cli/tools/installer.rs +++ b/cli/tools/installer.rs @@ -268,9 +268,9 @@ pub async fn install_command( // ensure the module is cached let factory = CliFactory::from_flags(flags.clone())?; - let mut module_graph_preparer = - factory.create_main_module_graph_preparer().await?; - module_graph_preparer + factory + .main_module_graph_container() + .await? .load_and_type_check_files(&[install_flags_global.module_url.clone()]) .await?; diff --git a/cli/tools/jupyter/mod.rs b/cli/tools/jupyter/mod.rs index 32bf9680e06b5..da1c4bc4d8f13 100644 --- a/cli/tools/jupyter/mod.rs +++ b/cli/tools/jupyter/mod.rs @@ -1,7 +1,5 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -use std::sync::Arc; - use crate::args::Flags; use crate::args::JupyterFlags; use crate::ops; @@ -20,7 +18,6 @@ use deno_core::resolve_url_or_path; use deno_core::serde::Deserialize; use deno_core::serde_json; use deno_core::url::Url; -use deno_graph::ModuleGraph; use deno_runtime::deno_io::Stdio; use deno_runtime::deno_io::StdioPipe; use deno_runtime::permissions::Permissions; @@ -94,7 +91,6 @@ pub async fn kernel( .create_custom_worker( WorkerExecutionMode::Jupyter, main_module.clone(), - Arc::new(ModuleGraph::new(cli_options.graph_kind())), permissions, vec![ ops::jupyter::deno_jupyter::init_ops(stdio_tx.clone()), diff --git a/cli/tools/repl/mod.rs b/cli/tools/repl/mod.rs index 7763c97d26d3e..d1c1cab7137e8 100644 --- a/cli/tools/repl/mod.rs +++ b/cli/tools/repl/mod.rs @@ -13,7 +13,6 @@ use deno_core::error::AnyError; use deno_core::futures::StreamExt; use deno_core::serde_json; use deno_core::unsync::spawn_blocking; -use deno_graph::ModuleGraph; use deno_runtime::permissions::Permissions; use deno_runtime::permissions::PermissionsContainer; use deno_runtime::WorkerExecutionMode; @@ -174,7 +173,6 @@ pub async fn run(flags: Flags, repl_flags: ReplFlags) -> Result { .create_custom_worker( WorkerExecutionMode::Repl, main_module.clone(), - Arc::new(ModuleGraph::new(cli_options.graph_kind())), permissions, vec![crate::ops::testing::deno_test::init_ops(test_event_sender)], Default::default(), diff --git a/cli/tools/run/mod.rs b/cli/tools/run/mod.rs index 51b1dbf9816b7..9f4bfeb96431c 100644 --- a/cli/tools/run/mod.rs +++ b/cli/tools/run/mod.rs @@ -1,10 +1,8 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. use std::io::Read; -use std::sync::Arc; use deno_core::error::AnyError; -use deno_graph::ModuleGraph; use deno_runtime::permissions::Permissions; use deno_runtime::permissions::PermissionsContainer; use deno_runtime::WorkerExecutionMode; @@ -71,12 +69,7 @@ To grant permissions, set them before the script argument. For example: )?); let worker_factory = factory.create_cli_main_worker_factory().await?; let mut worker = worker_factory - .create_main_worker( - mode, - main_module, - Arc::new(ModuleGraph::new(cli_options.graph_kind())), - permissions, - ) + .create_main_worker(mode, main_module, permissions) .await?; let exit_code = worker.run().await?; @@ -106,12 +99,7 @@ pub async fn run_from_stdin(flags: Flags) -> Result { }); let mut worker = worker_factory - .create_main_worker( - WorkerExecutionMode::Run, - main_module, - Arc::new(ModuleGraph::new(cli_options.graph_kind())), - permissions, - ) + .create_main_worker(WorkerExecutionMode::Run, main_module, permissions) .await?; let exit_code = worker.run().await?; Ok(exit_code) @@ -149,12 +137,7 @@ async fn run_with_watch( let mut worker = factory .create_cli_main_worker_factory() .await? - .create_main_worker( - mode, - main_module, - Arc::new(ModuleGraph::new(cli_options.graph_kind())), - permissions, - ) + .create_main_worker(mode, main_module, permissions) .await?; if watch_flags.hmr { @@ -203,12 +186,7 @@ pub async fn eval_command( )?); let worker_factory = factory.create_cli_main_worker_factory().await?; let mut worker = worker_factory - .create_main_worker( - WorkerExecutionMode::Eval, - main_module, - Arc::new(ModuleGraph::new(cli_options.graph_kind())), - permissions, - ) + .create_main_worker(WorkerExecutionMode::Eval, main_module, permissions) .await?; let exit_code = worker.run().await?; Ok(exit_code) diff --git a/cli/tools/test/mod.rs b/cli/tools/test/mod.rs index 4d04bd3c81e41..2cc51b4d32e82 100644 --- a/cli/tools/test/mod.rs +++ b/cli/tools/test/mod.rs @@ -10,8 +10,8 @@ use crate::factory::CliFactory; use crate::factory::CliFactoryBuilder; use crate::file_fetcher::File; use crate::file_fetcher::FileFetcher; +use crate::graph_container::MainModuleGraphContainer; use crate::graph_util::has_graph_root_local_dependent_changed; -use crate::module_load_preparer::MainModuleGraphPreparer; use crate::ops; use crate::util::file_watcher; use crate::util::fs::collect_specifiers; @@ -52,7 +52,6 @@ use deno_core::v8; use deno_core::ModuleSpecifier; use deno_core::OpState; use deno_core::PollEventLoopOptions; -use deno_graph::ModuleGraph; use deno_runtime::deno_io::Stdio; use deno_runtime::deno_io::StdioPipe; use deno_runtime::fmt_errors::format_js_error; @@ -579,7 +578,6 @@ fn get_test_reporter(options: &TestSpecifiersOptions) -> Box { async fn configure_main_worker( worker_factory: Arc, specifier: &Url, - module_graph: Arc, permissions: Permissions, worker_sender: TestEventWorkerSender, options: &TestSpecifierOptions, @@ -588,7 +586,6 @@ async fn configure_main_worker( .create_custom_worker( WorkerExecutionMode::Test, specifier.clone(), - module_graph, PermissionsContainer::new(permissions), vec![ops::testing::deno_test::init_ops(worker_sender.sender)], Stdio { @@ -634,7 +631,6 @@ pub async fn test_specifier( worker_factory: Arc, permissions: Permissions, specifier: ModuleSpecifier, - module_graph: Arc, worker_sender: TestEventWorkerSender, fail_fast_tracker: FailFastTracker, options: TestSpecifierOptions, @@ -645,7 +641,6 @@ pub async fn test_specifier( let (coverage_collector, mut worker) = configure_main_worker( worker_factory, &specifier, - module_graph, permissions, worker_sender, &options, @@ -1311,7 +1306,7 @@ async fn fetch_inline_files( /// Type check a collection of module and document specifiers. pub async fn check_specifiers( file_fetcher: &FileFetcher, - main_graph_preparer: &mut MainModuleGraphPreparer, + main_graph_container: &Arc, specifiers: Vec<(ModuleSpecifier, TestMode)>, ) -> Result<(), AnyError> { let inline_files = fetch_inline_files( @@ -1339,7 +1334,7 @@ pub async fn check_specifiers( file_fetcher.insert_memory_files(file); } - main_graph_preparer.check_specifiers(&specifiers).await?; + main_graph_container.check_specifiers(&specifiers).await?; } let module_specifiers = specifiers @@ -1353,7 +1348,7 @@ pub async fn check_specifiers( }) .collect::>(); - main_graph_preparer + main_graph_container .check_specifiers(&module_specifiers) .await?; @@ -1367,7 +1362,6 @@ async fn test_specifiers( worker_factory: Arc, permissions: &Permissions, specifiers: Vec, - module_graph: Arc, options: TestSpecifiersOptions, ) -> Result<(), AnyError> { let specifiers = if let Some(seed) = options.specifier.shuffle { @@ -1398,13 +1392,11 @@ async fn test_specifiers( let worker_sender = test_event_sender_factory.worker(); let fail_fast_tracker = fail_fast_tracker.clone(); let specifier_options = options.specifier.clone(); - let module_graph = module_graph.clone(); spawn_blocking(move || { create_and_run_current_thread(test_specifier( worker_factory, permissions, specifier, - module_graph, worker_sender, fail_fast_tracker, specifier_options, @@ -1723,12 +1715,11 @@ pub async fn run_tests( return Err(generic_error("No test modules found")); } - let mut main_graph_preparer = - factory.create_main_module_graph_preparer().await?; + let main_graph_container = factory.main_module_graph_container().await?; check_specifiers( file_fetcher, - &mut main_graph_preparer, + main_graph_container, specifiers_with_mode.clone(), ) .await?; @@ -1750,7 +1741,6 @@ pub async fn run_tests( _ => Some(s), }) .collect(), - Arc::new(main_graph_preparer.into_graph()), TestSpecifiersOptions { cwd: Url::from_directory_path(cli_options.initial_cwd()).map_err( |_| { @@ -1879,11 +1869,11 @@ pub async fn run_tests_with_watch( .filter(|(specifier, _)| test_modules_to_reload.contains(specifier)) .collect::>(); - let mut main_graph_preparer = - factory.create_main_module_graph_preparer().await?; + let main_graph_container = + factory.main_module_graph_container().await?; check_specifiers( file_fetcher, - &mut main_graph_preparer, + main_graph_container, specifiers_with_mode.clone(), ) .await?; @@ -1902,7 +1892,6 @@ pub async fn run_tests_with_watch( _ => Some(s), }) .collect(), - Arc::new(main_graph_preparer.into_graph()), TestSpecifiersOptions { cwd: Url::from_directory_path(cli_options.initial_cwd()).map_err( |_| { diff --git a/cli/worker.rs b/cli/worker.rs index ecee96a6222c5..ad24fc673b344 100644 --- a/cli/worker.rs +++ b/cli/worker.rs @@ -22,7 +22,6 @@ use deno_core::ModuleLoader; use deno_core::PollEventLoopOptions; use deno_core::SharedArrayBufferStore; use deno_core::SourceMapGetter; -use deno_graph::ModuleGraph; use deno_lockfile::Lockfile; use deno_runtime::code_cache; use deno_runtime::deno_broadcast_channel::InMemoryBroadcastChannel; @@ -67,7 +66,6 @@ pub struct ModuleLoaderAndSourceMapGetter { pub trait ModuleLoaderFactory: Send + Sync { fn create_for_main( &self, - main_module_graph: Arc, root_permissions: PermissionsContainer, dynamic_permissions: PermissionsContainer, ) -> ModuleLoaderAndSourceMapGetter; @@ -459,14 +457,12 @@ impl CliMainWorkerFactory { &self, mode: WorkerExecutionMode, main_module: ModuleSpecifier, - main_module_graph: Arc, permissions: PermissionsContainer, ) -> Result { self .create_custom_worker( mode, main_module, - main_module_graph, permissions, vec![], Default::default(), @@ -478,7 +474,6 @@ impl CliMainWorkerFactory { &self, mode: WorkerExecutionMode, main_module: ModuleSpecifier, - main_module_graph: Arc, permissions: PermissionsContainer, custom_extensions: Vec, stdio: deno_runtime::deno_io::Stdio, @@ -559,11 +554,9 @@ impl CliMainWorkerFactory { let ModuleLoaderAndSourceMapGetter { module_loader, source_map_getter, - } = shared.module_loader_factory.create_for_main( - main_module_graph, - PermissionsContainer::allow_all(), - permissions.clone(), - ); + } = shared + .module_loader_factory + .create_for_main(PermissionsContainer::allow_all(), permissions.clone()); let maybe_inspector_server = shared.maybe_inspector_server.clone(); let create_web_worker_cb = diff --git a/cli/workers/mod.rs b/cli/workers/mod.rs deleted file mode 100644 index b2330c24ec9b4..0000000000000 --- a/cli/workers/mod.rs +++ /dev/null @@ -1,7 +0,0 @@ -// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. - -mod module_graph_container; -mod module_loader; - -pub use module_graph_container::MainModuleGraphContainer; -pub use module_loader::CliModuleLoaderFactory; From 85d311e887f3f8dc971f85bee249deb739f3a2ce Mon Sep 17 00:00:00 2001 From: David Sherret Date: Fri, 3 May 2024 21:14:26 -0400 Subject: [PATCH 6/8] fix test --- tests/specs/node/worker_threads_cache/__test__.jsonc | 3 ++- tests/specs/node/worker_threads_cache/main.ts | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/specs/node/worker_threads_cache/__test__.jsonc b/tests/specs/node/worker_threads_cache/__test__.jsonc index fc471ddcef8cc..a47fed572dcd2 100644 --- a/tests/specs/node/worker_threads_cache/__test__.jsonc +++ b/tests/specs/node/worker_threads_cache/__test__.jsonc @@ -1,4 +1,5 @@ { + "tempDir": true, "args": "run -A main.ts", - "out": "main.out" + "output": "main.out" } diff --git a/tests/specs/node/worker_threads_cache/main.ts b/tests/specs/node/worker_threads_cache/main.ts index 65a4dd239d175..9703ac8f63ae0 100644 --- a/tests/specs/node/worker_threads_cache/main.ts +++ b/tests/specs/node/worker_threads_cache/main.ts @@ -8,6 +8,6 @@ const i = await import(path.href); console.log(i); if (isMainThread) { - const worker = new Worker(new URL("test.mjs", import.meta.url)); + const worker = new Worker(new URL("main.ts", import.meta.url)); worker.on("message", (msg) => console.log(msg)); } From a4252107e6c7b13f1ce3ca6fa02583b0956242f5 Mon Sep 17 00:00:00 2001 From: Divy Srivastava Date: Mon, 6 May 2024 13:05:04 +0530 Subject: [PATCH 7/8] update expectation --- tests/wpt/runner/expectation.json | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/wpt/runner/expectation.json b/tests/wpt/runner/expectation.json index f20cd78f11026..83ba128fbc37b 100644 --- a/tests/wpt/runner/expectation.json +++ b/tests/wpt/runner/expectation.json @@ -8698,9 +8698,7 @@ "blob-url.any.worker-module.html": [ "Revoking a blob URL immediately after calling import will not fail" ], - "blob-url-workers.window.html": [ - "A revoked blob URL will not resolve in a worker even if it's in the window's module graph" - ], + "blob-url-workers.window.html": [], "microtasks": { "basic.any.html": [ "import() should not drain the microtask queue if it fails during specifier resolution", From 02950d70bd27ab7670058901340004c7a8d712bd Mon Sep 17 00:00:00 2001 From: Divy Srivastava Date: Thu, 16 May 2024 10:37:43 +0530 Subject: [PATCH 8/8] fixes --- cli/module_loader.rs | 16 +++++++++++++--- tests/wpt/suite | 2 +- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/cli/module_loader.rs b/cli/module_loader.rs index 1abc59ae2c77f..9a8441ccd9979 100644 --- a/cli/module_loader.rs +++ b/cli/module_loader.rs @@ -16,10 +16,12 @@ use crate::cache::ModuleInfoCache; use crate::cache::ParsedSourceCache; use crate::emit::Emitter; use crate::factory::CliFactory; +use crate::graph_container::MainModuleGraphContainer; +use crate::graph_container::ModuleGraphContainer; +use crate::graph_container::ModuleGraphUpdatePermit; use crate::graph_util::graph_lock_or_exit; use crate::graph_util::CreateGraphOptions; use crate::graph_util::ModuleGraphBuilder; -use crate::graph_util::ModuleGraphContainer; use crate::node; use crate::resolver::CliGraphResolver; use crate::resolver::CliNodeResolver; @@ -30,6 +32,7 @@ use crate::tools::check::TypeChecker; use crate::util::progress_bar::ProgressBar; use crate::util::text_encoding::code_without_source_map; use crate::util::text_encoding::source_map_from_code; +use crate::worker::ModuleLoaderAndSourceMapGetter; use crate::worker::ModuleLoaderFactory; use deno_ast::MediaType; @@ -85,12 +88,19 @@ pub async fn load_top_level_deps(factory: &CliFactory) -> Result<(), AnyError> { entry.value.cloned() } }) - .collect(); + .collect::>(); + let mut graph_permit = factory + .main_module_graph_container() + .await? + .acquire_update_permit() + .await; + let graph = graph_permit.graph_mut(); factory .module_load_preparer() .await? .prepare_module_load( - roots, + graph, + &roots, false, factory.cli_options().ts_type_lib_window(), deno_runtime::permissions::PermissionsContainer::allow_all(), diff --git a/tests/wpt/suite b/tests/wpt/suite index daa07cf3c4765..5e8f71d73049d 160000 --- a/tests/wpt/suite +++ b/tests/wpt/suite @@ -1 +1 @@ -Subproject commit daa07cf3c47652ed67e637f2a39bbc34f91cfe10 +Subproject commit 5e8f71d73049d4fca2a8cbc62d40e821400f1624