diff --git a/Cargo.lock b/Cargo.lock index 9726f7a8da8b..b6be51c6c2f1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4721,6 +4721,7 @@ dependencies = [ "swc_node_base", "swc_plugin_proxy", "testing", + "tokio", "tracing", "wasmer", "wasmer-cache", @@ -5844,6 +5845,7 @@ dependencies = [ "pin-project", "rand", "serde", + "serde_cbor", "serde_derive", "serde_json", "serde_yaml", diff --git a/crates/swc/src/plugin.rs b/crates/swc/src/plugin.rs index fc8c6a13fd86..38d645b1f96e 100644 --- a/crates/swc/src/plugin.rs +++ b/crates/swc/src/plugin.rs @@ -100,6 +100,15 @@ impl RustPlugins { .expect("plugin module should be loaded"); let plugin_name = plugin_module_bytes.get_module_name().to_string(); + let runtime = swc_plugin_runner::wasix_runtime::build_wasi_runtime( + crate::config::PLUGIN_MODULE_CACHE + .inner + .get() + .unwrap() + .lock() + .get_fs_cache_root() + .map(|v| std::path::PathBuf::from(v)), + ); let mut transform_plugin_executor = swc_plugin_runner::create_plugin_transform_executor( &self.source_map, @@ -107,6 +116,7 @@ impl RustPlugins { &self.metadata_context, plugin_module_bytes, Some(p.1), + runtime, ); let span = tracing::span!( diff --git a/crates/swc_core/Cargo.toml b/crates/swc_core/Cargo.toml index ef84c877d5b0..dc82b2ec44ee 100644 --- a/crates/swc_core/Cargo.toml +++ b/crates/swc_core/Cargo.toml @@ -206,6 +206,9 @@ plugin_transform_host_native_filesystem_cache = [ "swc_plugin_runner/filesystem_cache", ] +plugin_transform_host_native_shared_runtime = [ + "swc_plugin_runner/plugin_transform_host_native_shared_runtime", +] ### Internal features that public features are relying on. ### This is not supposed to be used directly, and does not gaurantee ### stability across each versions. diff --git a/crates/swc_plugin_runner/Cargo.toml b/crates/swc_plugin_runner/Cargo.toml index e49e6c732499..43703edfa55d 100644 --- a/crates/swc_plugin_runner/Cargo.toml +++ b/crates/swc_plugin_runner/Cargo.toml @@ -30,6 +30,11 @@ plugin_transform_host_native = [ "wasmer-wasix/host-threads", "wasmer-compiler-cranelift/default", ] +plugin_transform_host_native_shared_runtime = [ + "tokio", + "wasmer-wasix/webc_runner", +] + # Supports a cache allow to store compiled bytecode into filesystem location. # This feature implies in-memory cache support. This is not supported on wasm32 target. filesystem_cache = ["wasmer-cache"] @@ -51,6 +56,7 @@ once_cell = "1.10.0" parking_lot = "0.12.0" serde = { version = "1.0.126", features = ["derive"] } serde_json = "1.0.64" +tokio = { version = "1", default-features = false, optional = true } tracing = "0.1.32" wasmer = { version = "3.3.0", default-features = false } wasmer-wasix = { version = "0.4.0", default-features = false } diff --git a/crates/swc_plugin_runner/benches/ecma_invoke.rs b/crates/swc_plugin_runner/benches/ecma_invoke.rs index 714e5ef893c9..99d3eade42e7 100644 --- a/crates/swc_plugin_runner/benches/ecma_invoke.rs +++ b/crates/swc_plugin_runner/benches/ecma_invoke.rs @@ -91,6 +91,7 @@ fn bench_transform(b: &mut Bencher, plugin_dir: &Path) { )), Box::new(plugin_module.clone()), None, + None, ); let experimental_metadata: VersionedSerializable> = diff --git a/crates/swc_plugin_runner/src/cache.rs b/crates/swc_plugin_runner/src/cache.rs index b330067ca3af..9d2da3225242 100644 --- a/crates/swc_plugin_runner/src/cache.rs +++ b/crates/swc_plugin_runner/src/cache.rs @@ -35,6 +35,8 @@ const MODULE_SERIALIZATION_VERSION: &str = "v6"; #[derive(Default)] pub struct PluginModuleCacheInner { + #[cfg(all(not(target_arch = "wasm32"), feature = "filesystem_cache"))] + fs_cache_root: Option, #[cfg(all(not(target_arch = "wasm32"), feature = "filesystem_cache"))] fs_cache_store: Option, // Stores the string representation of the hash of the plugin module to store into @@ -51,6 +53,13 @@ pub struct PluginModuleCacheInner { } impl PluginModuleCacheInner { + pub fn get_fs_cache_root(&self) -> Option { + #[cfg(all(not(target_arch = "wasm32"), feature = "filesystem_cache"))] + return self.fs_cache_root.clone(); + + None + } + /// Check if the cache contains bytes for the corresponding key. pub fn contains(&self, key: &str) -> bool { let is_in_cache = self.memory_cache_store.contains_key(key) @@ -183,6 +192,8 @@ impl PluginModuleCache { fs_cache_store_root: &Option, ) -> PluginModuleCacheInner { PluginModuleCacheInner { + #[cfg(all(not(target_arch = "wasm32"), feature = "filesystem_cache"))] + fs_cache_root: fs_cache_store_root.clone(), #[cfg(all(not(target_arch = "wasm32"), feature = "filesystem_cache"))] fs_cache_store: if enable_fs_cache_store { create_filesystem_cache(fs_cache_store_root) diff --git a/crates/swc_plugin_runner/src/lib.rs b/crates/swc_plugin_runner/src/lib.rs index 22e12038d7d1..bd307c3a9843 100644 --- a/crates/swc_plugin_runner/src/lib.rs +++ b/crates/swc_plugin_runner/src/lib.rs @@ -13,6 +13,7 @@ mod imported_fn; mod memory_interop; pub mod plugin_module_bytes; mod transform_executor; +pub mod wasix_runtime; use plugin_module_bytes::PluginModuleBytes; @@ -26,6 +27,7 @@ pub fn create_plugin_transform_executor( metadata_context: &Arc, plugin_module: Box, plugin_config: Option, + runtime: Option>, ) -> TransformExecutor { TransformExecutor::new( plugin_module, @@ -33,6 +35,7 @@ pub fn create_plugin_transform_executor( unresolved_mark, metadata_context, plugin_config, + runtime, ) } @@ -43,6 +46,7 @@ pub fn create_plugin_transform_executor( metadata_context: &Arc, plugin_module: Box, plugin_config: Option, + runtime: Option<()>, ) -> TransformExecutor { unimplemented!("Transform plugin cannot be used without serialization support") } diff --git a/crates/swc_plugin_runner/src/transform_executor.rs b/crates/swc_plugin_runner/src/transform_executor.rs index e6164cd438c6..9dfd71194365 100644 --- a/crates/swc_plugin_runner/src/transform_executor.rs +++ b/crates/swc_plugin_runner/src/transform_executor.rs @@ -14,7 +14,7 @@ use swc_common::{ SourceMap, }; use wasmer::{AsStoreMut, FunctionEnv, Instance, Store, TypedFunction}; -use wasmer_wasix::{default_fs_backing, is_wasi_module, WasiEnv, WasiFunctionEnv}; +use wasmer_wasix::{default_fs_backing, is_wasi_module, WasiEnv, WasiFunctionEnv, WasiRuntime}; use crate::plugin_module_bytes::PluginModuleBytes; #[cfg(feature = "__rkyv")] @@ -119,13 +119,9 @@ impl PluginTransformState { guest_program_ptr.1, )?; - /* [Note]: currently this is disabled: this cleanup is for the multi-threaded - * wasi environment as well as cleaning up file handles which is for the cases - * running wasi binary as standalone. SWC doesn't need neither currently, will - * revisit once we support multithreaded plugin. if let Some(wasi_env) = &self.wasi_env { wasi_env.cleanup(&mut self.store, None); - }*/ + } ret } @@ -177,6 +173,7 @@ pub struct TransformExecutor { metadata_context: Arc, plugin_config: Option, module_bytes: Box, + runtime: Option>, } #[cfg(feature = "__rkyv")] @@ -191,6 +188,7 @@ impl TransformExecutor { unresolved_mark: &swc_common::Mark, metadata_context: &Arc, plugin_config: Option, + runtime: Option>, ) -> Self { Self { source_map: source_map.clone(), @@ -198,6 +196,7 @@ impl TransformExecutor { metadata_context: metadata_context.clone(), plugin_config, module_bytes, + runtime, } } @@ -260,7 +259,11 @@ impl TransformExecutor { // TODO: wasm host native runtime throws 'Memory should be set on `WasiEnv` // first' let (instance, wasi_env) = if is_wasi_module(&module) { - let builder = WasiEnv::builder(self.module_bytes.get_module_name()); + let mut builder = WasiEnv::builder(self.module_bytes.get_module_name()); + + if let Some(runtime) = &self.runtime { + builder.set_runtime(runtime.clone()); + } // Implicitly enable filesystem access for the wasi plugin to cwd. // @@ -374,252 +377,3 @@ impl TransformExecutor { transform_state.run(program, self.unresolved_mark, should_enable_comments_proxy) } } - -/* -/// A struct encapsule executing a plugin's transform interop to its teardown -pub struct TransformExecutor { - // Main transform interface plugin exports - exported_plugin_transform: TypedFunction<(u32, u32, u32, u32), u32>, - // `__free` function automatically exported via swc_plugin sdk to allow deallocation in guest - // memory space - exported_plugin_free: TypedFunction<(u32, u32), u32>, - // `__alloc` function automatically exported via swc_plugin sdk to allow allocation in guest - // memory space - exported_plugin_alloc: TypedFunction, - wasi_env: Option, - instance: Instance, - store: Store, - // Reference to the pointers successfully allocated which'll be freed by Drop. - allocated_ptr_vec: Vec<(u32, u32)>, - transform_result: Arc>>, - // diagnostic metadata for the swc_core plugin binary uses. - pub plugin_core_diag: PluginCorePkgDiagnostics, -} - -#[cfg(feature = "__rkyv")] -impl TransformExecutor { - #[tracing::instrument( - level = "info", - skip(source_map, metadata_context, plugin_config, module_bytes) - )] - pub fn new( - source_map: &Arc, - metadata_context: &Arc, - plugin_config: Option, - module_bytes: &dyn PluginModuleBytes, - ) -> Self { - unimplemented!() - } - - - /// Creates a transform executor from a raw bytes. - #[tracing::instrument( - level = "info", - skip(source_map, metadata_context, plugin_config, plugin_module) - )] - pub fn from_raw_bytes( - source_map: &Arc, - metadata_context: &Arc, - plugin_config: Option, - plugin_module: &RawPluginModuleBytes, - ) -> Result { - let (store, module) = plugin_module.compile_module()?; - - TransformExecutor::from_module( - source_map, - metadata_context, - plugin_config, - &plugin_module.plugin_name, - store, - module, - ) - } - - /// Creates a transform executor from a bytes seriaized by runtime (wasmer). - #[tracing::instrument( - level = "info", - skip(source_map, metadata_context, plugin_config, plugin_module) - )] - pub fn from_serialized_bytes( - source_map: &Arc, - metadata_context: &Arc, - plugin_config: Option, - plugin_module: &SerializedPluginModuleBytes, - ) -> Result { - let (store, module) = plugin_module.compile_module()?; - - TransformExecutor::from_module( - source_map, - metadata_context, - plugin_config, - &plugin_module.plugin_name, - store, - module, - ) - } - - /// Creates a transform executor from an already compiled module. - #[tracing::instrument( - level = "info", - skip(source_map, metadata_context, plugin_config, store, module) - )] - pub fn from_module( - source_map: &Arc, - metadata_context: &Arc, - plugin_config: Option, - plugin_name: &str, - store: Store, - module: Module, - ) -> Result { - let (instance, transform_result, diagnostics_buffer, wasi_env) = - crate::load_plugin::load_plugin( - &mut store, - &mut module, - source_map, - metadata_context, - plugin_config, - plugin_name, - )?; - - Ok(TransformExecutor { - exported_plugin_transform: instance - .exports - .get_typed_function(&store, "__transform_plugin_process_impl")?, - exported_plugin_free: instance.exports.get_typed_function(&store, "__free")?, - exported_plugin_alloc: instance.exports.get_typed_function(&store, "__alloc")?, - instance, - store, - wasi_env, - allocated_ptr_vec: Vec::with_capacity(3), - transform_result, - plugin_core_diag: diagnostics_buffer, - }) - } - - /// Copy host's serialized bytes into guest (plugin)'s allocated memory. - /// Once transformation completes, host should free allocated memory. - #[tracing::instrument(level = "info", skip_all)] - fn write_bytes_into_guest( - &mut self, - serialized_bytes: &PluginSerializedBytes, - ) -> Result<(u32, u32), Error> { - let memory = self.instance.exports.get_memory("memory")?; - - let ptr = write_into_memory_view( - memory, - &mut self.store.as_store_mut(), - serialized_bytes, - |s, serialized_len| { - self.exported_plugin_alloc - .call(s, serialized_len.try_into().expect("booo")) - .unwrap_or_else(|_| { - panic!( - "Should able to allocate memory for the size of {}", - serialized_len - ) - }) - }, - ); - - self.allocated_ptr_vec.push(ptr); - Ok(ptr) - } - - /// Copy guest's memory into host, construct serialized struct from raw - /// bytes. - fn read_transformed_result_bytes_from_guest( - &mut self, - returned_ptr_result: u32, - ) -> Result { - let transformed_result = &(*self.transform_result.lock()); - let ret = PluginSerializedBytes::from_slice(&transformed_result[..]); - - if returned_ptr_result == 0 { - Ok(ret) - } else { - let err: PluginError = ret.deserialize()?.into_inner(); - match err { - PluginError::SizeInteropFailure(msg) => Err(anyhow!( - "Failed to convert pointer size to calculate: {}", - msg - )), - PluginError::Deserialize(msg) | PluginError::Serialize(msg) => { - Err(anyhow!("{}", msg)) - } - _ => Err(anyhow!( - "Unexpected error occurred while running plugin transform" - )), - } - } - } - - /** - * Check compile-time version of AST schema between the plugin and - * the host. Returns true if it's compatible, false otherwise. - * - * Host should appropriately handle if plugin is not compatible to the - * current runtime. - */ - #[allow(unreachable_code)] - pub fn is_transform_schema_compatible(&mut self) -> Result { - #[cfg(any( - feature = "plugin_transform_schema_v1", - feature = "plugin_transform_schema_vtest" - ))] - return { - let host_schema_version = PLUGIN_TRANSFORM_AST_SCHEMA_VERSION; - - // TODO: this is incomplete - if host_schema_version >= self.plugin_core_diag.ast_schema_version { - Ok(true) - } else { - Ok(false) - } - }; - - #[cfg(not(all( - feature = "plugin_transform_schema_v1", - feature = "plugin_transform_schema_vtest" - )))] - anyhow::bail!( - "Plugin runner cannot detect plugin's schema version. Ensure host is compiled with \ - proper versions" - ) - } - - #[tracing::instrument(level = "info", skip_all)] - pub fn transform( - &mut self, - program: &PluginSerializedBytes, - unresolved_mark: swc_common::Mark, - should_enable_comments_proxy: bool, - ) -> Result { - let should_enable_comments_proxy = u32::from(should_enable_comments_proxy); - let guest_program_ptr = self.write_bytes_into_guest(program)?; - - let result = self.exported_plugin_transform.call( - &mut self.store, - guest_program_ptr.0, - guest_program_ptr.1, - unresolved_mark.as_u32(), - should_enable_comments_proxy, - )?; - - self.read_transformed_result_bytes_from_guest(result) - } -} - -impl Drop for TransformExecutor { - fn drop(&mut self) { - for ptr in self.allocated_ptr_vec.iter() { - self.exported_plugin_free - .call(&mut self.store, ptr.0, ptr.1) - .expect("Failed to free memory allocated in the plugin"); - } - - if let Some(wasi_env) = &self.wasi_env { - wasi_env.cleanup(&mut self.store, None); - } - } -} - */ diff --git a/crates/swc_plugin_runner/src/wasix_runtime.rs b/crates/swc_plugin_runner/src/wasix_runtime.rs new file mode 100644 index 000000000000..2ca7b673c5e4 --- /dev/null +++ b/crates/swc_plugin_runner/src/wasix_runtime.rs @@ -0,0 +1,44 @@ +use std::{path::PathBuf, sync::Arc}; + +use wasmer_wasix::WasiRuntime; + +/// Construct a runtime for the wasix engine depends on the compilation +/// features. +/// +/// This is mainly for the case if a host already sets up its runtime, which +/// makes wasix initialization fails due to conflicting runtime. When specified, +/// instead of using default runtime it'll try to use shared one. +pub fn build_wasi_runtime( + fs_cache_path: Option, +) -> Option> { + #[cfg(not(feature = "plugin_transform_host_native_shared_runtime"))] + return None; + + #[cfg(feature = "plugin_transform_host_native_shared_runtime")] + { + use wasmer_wasix::{ + runners::Runner, runtime::task_manager::tokio::TokioTaskManager, PluggableRuntime, + }; + + let tasks = TokioTaskManager::new(tokio::runtime::Handle::current()); + let mut rt = PluggableRuntime::new(Arc::new(tasks)); + + /* [TODO]: wasmer@4 + #[cfg(all(not(target_arch = "wasm32"), feature = "filesystem_cache"))] + let cache = if let Some(fs_cache_path) = fs_cache_path { + SharedCache::default().with_fallback(wasmer_cache::FileSystemCache::new(fs_cache_path)) + } else { + SharedCache::default().with_fallback(wasmer_wasix::runtime::module_cache::in_memory()) + }; + + #[cfg(not(feature = "filesystem_cache"))] + let cache = SharedCache::default().with_fallback(in_memory()); + */ + + rt.set_engine(Some(wasmer::Engine::default())); + //[TODO]: wasmer@4 + //rt.set_module_cache(cache); + + return Some(Arc::new(rt)); + } +} diff --git a/crates/swc_plugin_runner/tests/css_rkyv.rs b/crates/swc_plugin_runner/tests/css_rkyv.rs index a006cbac4ab8..16c77a79da51 100644 --- a/crates/swc_plugin_runner/tests/css_rkyv.rs +++ b/crates/swc_plugin_runner/tests/css_rkyv.rs @@ -107,6 +107,7 @@ fn invoke(input: PathBuf) -> Result<(), Error> { )), Box::new(PLUGIN_BYTES.clone()), Some(json!({ "pluginConfig": "testValue" })), + None, ); info!("Created transform executor"); @@ -158,6 +159,7 @@ fn invoke(input: PathBuf) -> Result<(), Error> { )), Box::new(PLUGIN_BYTES.clone()), Some(json!({ "pluginConfig": "testValue" })), + None, ); serialized_program = plugin_transform_executor @@ -175,6 +177,7 @@ fn invoke(input: PathBuf) -> Result<(), Error> { )), Box::new(PLUGIN_BYTES.clone()), Some(json!({ "pluginConfig": "testValue" })), + None, ); serialized_program = plugin_transform_executor diff --git a/crates/swc_plugin_runner/tests/ecma_integration.rs b/crates/swc_plugin_runner/tests/ecma_integration.rs index 7e5a23bf7535..0451be2dc1af 100644 --- a/crates/swc_plugin_runner/tests/ecma_integration.rs +++ b/crates/swc_plugin_runner/tests/ecma_integration.rs @@ -135,6 +135,7 @@ fn internal() -> Result<(), Error> { )), Box::new(PLUGIN_BYTES.clone()), Some(json!({ "pluginConfig": "testValue" })), + None, ); /* [TODO]: reenable this later @@ -200,6 +201,7 @@ fn internal() -> Result<(), Error> { )), Box::new(PLUGIN_BYTES.clone()), Some(json!({ "pluginConfig": "testValue" })), + None, ); plugin_transform_executor @@ -248,6 +250,7 @@ fn internal() -> Result<(), Error> { )), Box::new(PLUGIN_BYTES.clone()), Some(json!({ "pluginConfig": "testValue" })), + None, ); serialized_program = plugin_transform_executor @@ -265,6 +268,7 @@ fn internal() -> Result<(), Error> { )), Box::new(PLUGIN_BYTES.clone()), Some(json!({ "pluginConfig": "testValue" })), + None, ); serialized_program = plugin_transform_executor diff --git a/crates/swc_plugin_runner/tests/ecma_rkyv.rs b/crates/swc_plugin_runner/tests/ecma_rkyv.rs index 40b6702eaacb..7bddc212da43 100644 --- a/crates/swc_plugin_runner/tests/ecma_rkyv.rs +++ b/crates/swc_plugin_runner/tests/ecma_rkyv.rs @@ -117,6 +117,7 @@ fn internal(input: PathBuf) -> Result<(), Error> { )), Box::new(PLUGIN_BYTES.clone()), Some(json!({ "pluginConfig": "testValue" })), + None, ); info!("Created transform executor"); @@ -174,6 +175,7 @@ fn internal(input: PathBuf) -> Result<(), Error> { )), Box::new(PLUGIN_BYTES.clone()), Some(json!({ "pluginConfig": "testValue" })), + None, ); serialized_program = plugin_transform_executor @@ -191,6 +193,7 @@ fn internal(input: PathBuf) -> Result<(), Error> { )), Box::new(PLUGIN_BYTES.clone()), Some(json!({ "pluginConfig": "testValue" })), + None, ); serialized_program = plugin_transform_executor diff --git a/crates/swc_plugin_runner/tests/issues.rs b/crates/swc_plugin_runner/tests/issues.rs index b21bf25530e3..86e9af7ef755 100644 --- a/crates/swc_plugin_runner/tests/issues.rs +++ b/crates/swc_plugin_runner/tests/issues.rs @@ -134,6 +134,7 @@ fn issue_6404() -> Result<(), Error> { )), Box::new(plugin_module), Some(json!({ "pluginConfig": "testValue" })), + None, ); /* [TODO]: reenable this test