diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 2f988fc711..ef17bfe101 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -84,6 +84,7 @@ jobs: - nox-snapshot uses: fluencelabs/cli/.github/workflows/tests.yml@main with: + ref: up-spell-and-fixes nox-image: "${{ needs.nox-snapshot.outputs.nox-image }}" js-client: @@ -91,6 +92,7 @@ jobs: - nox-snapshot uses: fluencelabs/js-client/.github/workflows/tests.yml@master with: + ref: js-client-v0.8.4 nox-image: "${{ needs.nox-snapshot.outputs.nox-image }}" aqua: diff --git a/Cargo.lock b/Cargo.lock index 174527c578..5b836e4956 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1644,6 +1644,7 @@ dependencies = [ "air-interpreter-fs", "aquamarine", "base64 0.21.7", + "cid-utils", "config-utils", "connection-pool", "core-manager", @@ -1853,9 +1854,9 @@ dependencies = [ [[package]] name = "decider-distro" -version = "0.6.9" +version = "0.6.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "084ac8f880cfa9b885b38c4b10dd01c8bd6e5c79a689d59c760440a011cab425" +checksum = "5885bba1b2e16ebff830b6229ba4fc7c4df6eefe1cccdb37ec86c3d9886ddea1" dependencies = [ "built 0.7.1", "fluence-spell-dtos", @@ -2343,9 +2344,9 @@ dependencies = [ [[package]] name = "fluence-spell-distro" -version = "0.7.4" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4f7a6eb40ae202258affc0cde4ec51e4b49dfb01e44b2b01fe97352beaa46a1" +checksum = "0f52833758cb60847d144acf2236d2cb758a067cd90eb3905cb663093c4e81a7" dependencies = [ "built 0.7.1", "maplit", @@ -2353,9 +2354,9 @@ dependencies = [ [[package]] name = "fluence-spell-dtos" -version = "0.7.4" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13c65a23ae0bd9c838c3dba0ed167222e36026d7ce0f29f89faf5a47eff28148" +checksum = "e3794dff0e7e2851fee56651bf45f8d57c8be3c77c285a5f16fd86d4d6ed1037" dependencies = [ "eyre", "marine-rs-sdk 0.14.0", @@ -5912,7 +5913,9 @@ dependencies = [ "json-utils", "libipld", "log", + "maplit", "marine-it-parser 0.16.0", + "marine-module-info-parser 0.15.0", "parking_lot", "particle-args", "particle-execution", @@ -7284,6 +7287,7 @@ dependencies = [ "base64 0.21.7", "bs58", "bytesize", + "cid-utils", "clap 4.4.18", "clarity", "config", @@ -7299,6 +7303,7 @@ dependencies = [ "libp2p-connection-limits", "libp2p-metrics", "log", + "maplit", "num_cpus", "particle-protocol", "peer-metrics", diff --git a/Cargo.toml b/Cargo.toml index 26a9957ce5..c12cb329a3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -106,13 +106,14 @@ types = { path = "crates/types" } core-manager = { path = "crates/core-manager" } # spell -fluence-spell-dtos = "=0.7.4" -fluence-spell-distro = "=0.7.4" +fluence-spell-dtos = "=0.7.5" +fluence-spell-distro = "=0.7.5" # marine fluence-app-service = "0.35.0" marine-utils = "0.5.1" marine-it-parser = "0.16.0" +marine-module-info-parser = "0.15.0" # avm avm-server = "=0.35.0" @@ -177,7 +178,6 @@ ccp-shared = { git = "https://github.com/fluencelabs/capacity-commitment-prover/ ccp-rpc-client = { git = "https://github.com/fluencelabs/capacity-commitment-prover.git", branch = "main" } -# Enable a small amount of optimization in debug mode [profile.dev] opt-level = 0 diff --git a/crates/cid-utils/src/hash.rs b/crates/cid-utils/src/hash.rs index 8c029fbdb1..368c55ea2a 100644 --- a/crates/cid-utils/src/hash.rs +++ b/crates/cid-utils/src/hash.rs @@ -34,7 +34,7 @@ use crate::unixfs::Data as UnixFsMetadata; /// It should be used everywhere in the Fluence stack to produce the same CIDs. const CHUNK_SIZE: usize = 262144; -#[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[derive(Clone, PartialEq, Eq, Hash, Debug)] pub struct Hash(pub Cid); impl Hash { diff --git a/crates/created-swarm/Cargo.toml b/crates/created-swarm/Cargo.toml index 396c25d7eb..c3f956a153 100644 --- a/crates/created-swarm/Cargo.toml +++ b/crates/created-swarm/Cargo.toml @@ -21,6 +21,7 @@ toy-vms = { workspace = true } system-services = { workspace = true } tempfile = { workspace = true } core-manager = { workspace = true } +cid-utils = { workspace = true } fluence-keypair = { workspace = true } log = { workspace = true } diff --git a/crates/created-swarm/src/swarm.rs b/crates/created-swarm/src/swarm.rs index dca7cc8c0e..1bf53cabed 100644 --- a/crates/created-swarm/src/swarm.rs +++ b/crates/created-swarm/src/swarm.rs @@ -14,6 +14,7 @@ * limitations under the License. */ +use std::collections::HashMap; use std::convert::identity; use std::net::SocketAddr; use std::sync::Arc; @@ -31,6 +32,7 @@ use air_interpreter_fs::{air_interpreter_path, write_default_air_interpreter}; use aquamarine::{AVMRunner, AquamarineApi}; use aquamarine::{AquaRuntime, DataStoreConfig, VmConfig}; use base64::{engine::general_purpose::STANDARD as base64, Engine}; +use cid_utils::Hash; use core_manager::manager::DummyCoreManager; use fluence_libp2p::random_multiaddr::{create_memory_maddr, create_tcp_maddr}; use fluence_libp2p::Transport; @@ -267,6 +269,7 @@ pub struct SwarmConfig { pub builtins_dir: Option, pub spell_base_dir: Option, pub allowed_binaries: Vec, + pub allowed_effectors: HashMap>, pub enabled_system_services: Vec, pub extend_system_services: Vec, pub override_system_services_config: Option, @@ -295,6 +298,7 @@ impl SwarmConfig { builtins_dir: None, spell_base_dir: None, allowed_binaries: vec!["/usr/bin/ipfs".to_string(), "/usr/bin/curl".to_string()], + allowed_effectors: HashMap::new(), enabled_system_services: vec![], extend_system_services: vec![], override_system_services_config: None, @@ -402,6 +406,11 @@ pub async fn create_swarm_with_runtime( resolved.node_config.allowed_binaries = config.allowed_binaries.clone(); + let allowed_effectors = config.allowed_effectors.iter().map(|(cid, binaries)| { + (Hash::from_string(cid).unwrap(), binaries.clone()) + }).collect::<_>(); + resolved.node_config.allowed_effectors = allowed_effectors; + if let Some(config) = config.override_system_services_config.clone() { resolved.system_services = config; } diff --git a/crates/nox-tests/tests/effector/Cargo.lock b/crates/nox-tests/tests/effector/Cargo.lock new file mode 100644 index 0000000000..397bd9d84e --- /dev/null +++ b/crates/nox-tests/tests/effector/Cargo.lock @@ -0,0 +1,404 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "bumpalo" +version = "3.15.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ea184aa71bb362a1157c896979544cc23974e08fd265f29ea96b59f0b4a555b" + +[[package]] +name = "cc" +version = "1.0.86" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f9fa1897e4325be0d68d48df6aa1a71ac2ed4d27723887e7754192705350730" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "chrono" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5bc015644b92d5890fab7489e49d21f879d5c990186827d42ec511919404f38b" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "js-sys", + "num-traits", + "wasm-bindgen", + "windows-targets", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" + +[[package]] +name = "effector" +version = "0.1.0" +dependencies = [ + "marine-rs-sdk", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "itoa" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" + +[[package]] +name = "js-sys" +version = "0.3.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "406cda4b368d531c842222cf9d2600a9a4acce8d29423695379c6868a143a9ee" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.153" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" + +[[package]] +name = "log" +version = "0.4.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" + +[[package]] +name = "marine-call-parameters" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05495180730abae04abe209386ce367309a82110edb65fcdb1f3080f819bc1a0" +dependencies = [ + "marine-macro", + "marine-rs-sdk-main", + "serde", +] + +[[package]] +name = "marine-macro" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f502185316f584a9373cceb6ff24a11d260dfd39505c817056bc127cd1a96a08" +dependencies = [ + "marine-macro-impl", + "marine-rs-sdk-main", +] + +[[package]] +name = "marine-macro-impl" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e50fbc0e70ee4cde7802f0748acfb197d7770c7feffb980ce8c29bddd007519e" +dependencies = [ + "proc-macro2", + "quote", + "serde", + "serde_json", + "syn 1.0.109", +] + +[[package]] +name = "marine-rs-sdk" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f93d2bd852fea1fea8097c195044430347eda98fd6a3752119b549192d5ac4ba" +dependencies = [ + "marine-call-parameters", + "marine-macro", + "marine-rs-sdk-main", + "marine-timestamp-macro", + "serde", +] + +[[package]] +name = "marine-rs-sdk-main" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b79c165fc21438b069babeec5ae36ba0eade5e08fb1d92dabbe6b41014ce841" +dependencies = [ + "log", + "serde", +] + +[[package]] +name = "marine-timestamp-macro" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d03f267ac0a29f543ef12a1a519ff8d98e74ac66e1c580f2930d41ce2c50507d" +dependencies = [ + "chrono", + "quote", +] + +[[package]] +name = "num-traits" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da0df0e5185db44f69b44f26786fe401b6c293d1907744beaa7fa62b2e5a517a" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" + +[[package]] +name = "proc-macro2" +version = "1.0.78" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2422ad645d89c99f8f3e6b88a9fdeca7fabeac836b1002371c4367c8f984aae" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "ryu" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e86697c916019a8588c99b5fac3cead74ec0b4b819707a682fd4d23fa0ce1ba1" + +[[package]] +name = "serde" +version = "1.0.197" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fb1c873e1b9b056a4dc4c0c198b24c3ffa059243875552b2bd0933b1aee4ce2" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.197" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7eb0b34b42edc17f6b7cac84a52a1c5f0e1bb2227e997ca9011ea3dd34e8610b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.50", +] + +[[package]] +name = "serde_json" +version = "1.0.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5f09b1bd632ef549eaa9f60a1f8de742bdbc698e6cee2095fc84dde5f549ae0" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74f1bdc9872430ce9b75da68329d1c1746faf50ffac5f19e02b71e37ff881ffb" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" + +[[package]] +name = "wasm-bindgen" +version = "0.2.91" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1e124130aee3fb58c5bdd6b639a0509486b0338acaaae0c84a5124b0f588b7f" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.91" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9e7e1900c352b609c8488ad12639a311045f40a35491fb69ba8c12f758af70b" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn 2.0.50", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.91" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b30af9e2d358182b5c7449424f017eba305ed32a7010509ede96cdc4696c46ed" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.91" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "642f325be6301eb8107a83d12a8ac6c1e1c54345a7ef1a9261962dfefda09e66" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.50", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.91" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f186bd2dcf04330886ce82d6f33dd75a7bfcf69ecf5763b89fcde53b6ac9838" + +[[package]] +name = "windows-core" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.52.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d380ba1dc7187569a8a9e91ed34b8ccfc33123bbacb8c0aed2d1ad7f3ef2dc5f" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68e5dcfb9413f53afd9c8f86e56a7b4d86d9a2fa26090ea2dc9e40fba56c6ec6" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8dab469ebbc45798319e69eebf92308e541ce46760b49b18c6b3fe5e8965b30f" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a4e9b6a7cac734a8b4138a4e1044eac3404d8326b6c0f939276560687a033fb" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28b0ec9c422ca95ff34a78755cfa6ad4a51371da2a5ace67500cf7ca5f232c58" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704131571ba93e89d7cd43482277d6632589b18ecf4468f591fbae0a8b101614" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42079295511643151e98d61c38c0acc444e52dd42ab456f7ccfd5152e8ecf21c" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0770833d60a970638e989b3fa9fd2bb1aaadcf88963d1659fd7d9990196ed2d6" diff --git a/crates/nox-tests/tests/effector/Cargo.toml b/crates/nox-tests/tests/effector/Cargo.toml new file mode 100644 index 0000000000..8060e6f768 --- /dev/null +++ b/crates/nox-tests/tests/effector/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "effector" +version = "0.1.0" +edition = "2021" +authors = ["Fluence Labs"] + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +marine-rs-sdk = "0.14.0" + +[workspace] diff --git a/crates/nox-tests/tests/effector/artifacts/Config.toml b/crates/nox-tests/tests/effector/artifacts/Config.toml new file mode 100644 index 0000000000..39a358bf19 --- /dev/null +++ b/crates/nox-tests/tests/effector/artifacts/Config.toml @@ -0,0 +1,6 @@ +total_memory_limit = "Infinity" + +[[module]] + name = "effector" + [module.mounted_binaries] + ls = "/bin/ls" # will it work on all systems when nox runs? diff --git a/crates/nox-tests/tests/effector/artifacts/effector.wasm b/crates/nox-tests/tests/effector/artifacts/effector.wasm new file mode 100755 index 0000000000..c814672eb1 Binary files /dev/null and b/crates/nox-tests/tests/effector/artifacts/effector.wasm differ diff --git a/crates/nox-tests/tests/effector/src/main.rs b/crates/nox-tests/tests/effector/src/main.rs new file mode 100644 index 0000000000..0880192c9f --- /dev/null +++ b/crates/nox-tests/tests/effector/src/main.rs @@ -0,0 +1,43 @@ +use marine_rs_sdk::marine; +use marine_rs_sdk::module_manifest; +use marine_rs_sdk::MountedBinaryResult; + +module_manifest!(); + +fn main() { } + +#[marine] +pub struct Result { + success: bool, + result: Vec, + error: Vec, +} + +#[marine] +fn list_directory(path: String) -> Result { + let result = ls(vec![inject_vault_host_path(path)]); + if let Some(result) = result.into_std() { + match result { + Ok(out) => Result { error: vec![], result: vec![out], success: true }, + Err(err) => Result { error: vec![err.to_string()], result: vec![], success: false}, + } + } else { + Result { error: vec!["MountedBinaryResult::into_std return None".to_string()], result: vec![], success: true } + } +} + +#[marine] +#[host_import] +extern "C" { + fn ls(cmd: Vec) -> MountedBinaryResult; +} + +fn inject_vault_host_path(path: String) -> String { + let vault = "/tmp/vault"; + if let Some(stripped) = path.strip_prefix(&vault) { + let host_vault_path = std::env::var(vault).expect("vault must be mapped to /tmp/vault"); + format!("/{}/{}", host_vault_path, stripped) + } else { + path + } +} diff --git a/crates/nox-tests/tests/modules.rs b/crates/nox-tests/tests/modules.rs index 5b138d9e5a..5057dc3234 100644 --- a/crates/nox-tests/tests/modules.rs +++ b/crates/nox-tests/tests/modules.rs @@ -24,7 +24,11 @@ use service_modules::load_module; #[tokio::test] async fn test_add_module_mounted_binaries() { let swarms = make_swarms_with_cfg(1, |mut cfg| { - cfg.allowed_binaries = vec!["/usr/bin/curl".to_string()]; + cfg.allowed_effectors = hashmap! { + "bafkreiepzclggkt57vu7yrhxylfhaafmuogtqly7wel7ozl5k2ehkd44oe".to_string() => hashmap! { + "ls".to_string() => "/bin/ls".to_string() + } + }; cfg }) .await; @@ -32,11 +36,11 @@ async fn test_add_module_mounted_binaries() { let mut client = ConnectedClient::connect_to(swarms[0].multiaddr.clone()) .await .expect("connect client"); - let module = load_module("tests/tetraplets/artifacts", "tetraplets").expect("load module"); + let module = load_module("tests/effector/artifacts", "effector").expect("load module"); let config = json!( { - "name": "tetraplets", + "name": "effector", "mem_pages_count": 100, "logger_enabled": true, "wasi": { @@ -44,7 +48,7 @@ async fn test_add_module_mounted_binaries() { "preopened_files": vec!["/tmp"], "mapped_dirs": json!({}), }, - "mounted_binaries": json!({"cmd": "/usr/bin/curl"}) + "mounted_binaries": json!({"ls": "/bin/ls"}) }); let script = r#" @@ -73,13 +77,13 @@ async fn test_add_module_mounted_binaries() { } #[tokio::test] -async fn test_add_module_mounted_binaries_forbidden() { +async fn test_add_module_effectors_forbidden() { let swarms = make_swarms(1).await; let mut client = ConnectedClient::connect_to(swarms[0].multiaddr.clone()) .await .expect("connect client"); - let module = load_module("tests/tetraplets/artifacts", "tetraplets").expect("load module"); + let module = load_module("tests/effector/artifacts", "effector").expect("load module"); let config = json!( { @@ -112,7 +116,7 @@ async fn test_add_module_mounted_binaries_forbidden() { let response = client.execute_particle(script, data).await.unwrap(); if let Some(result) = response[0].as_str() { - let expected = "Local service error, ret_code is 1, error message is '\"Error: Config error: requested mounted binary /usr/bin/behbehbeh is forbidden on this host\\nForbiddenMountedBinary { forbidden_path: \\\"/usr/bin/behbehbeh\\\" }\"'"; + let expected = "Local service error, ret_code is 1, error message is '\"Error: Config error: requested module effector tetraplets with CID bafkreiepzclggkt57vu7yrhxylfhaafmuogtqly7wel7ozl5k2ehkd44oe is forbidden on this host\\nForbiddenEffector { module_name: \\\"tetraplets\\\", forbidden_cid: Hash(Cid(bafkreiepzclggkt57vu7yrhxylfhaafmuogtqly7wel7ozl5k2ehkd44oe)) }\"'"; assert_eq!(expected, result); } else { panic!("can't receive response from node"); diff --git a/crates/nox-tests/tests/network/network_explore.rs b/crates/nox-tests/tests/network/network_explore.rs index 42123755c0..f39dc8328e 100644 --- a/crates/nox-tests/tests/network/network_explore.rs +++ b/crates/nox-tests/tests/network/network_explore.rs @@ -190,8 +190,8 @@ async fn list_blueprints() { .wrap_err("connect client") .unwrap(); - let bytes = b"module"; - let module_hash = Hash::new(bytes).unwrap().to_string(); + let bytes = load_module("tests/file_share/artifacts", "file_share").expect("load module"); + let module_hash = Hash::new(&bytes).unwrap().to_string(); client .send_particle( r#" diff --git a/crates/nox-tests/tests/tetraplets/Cargo.toml b/crates/nox-tests/tests/tetraplets/Cargo.toml index bc2df72b8b..c9a21a3434 100644 --- a/crates/nox-tests/tests/tetraplets/Cargo.toml +++ b/crates/nox-tests/tests/tetraplets/Cargo.toml @@ -10,6 +10,8 @@ edition = "2021" marine-rs-sdk = "0.14.0" once_cell = "1.19.0" +[workspace] + [[bin]] name = "tetraplets" -path = "src/main.rs" \ No newline at end of file +path = "src/main.rs" diff --git a/crates/server-config/Cargo.toml b/crates/server-config/Cargo.toml index 9090ee5da3..0b26dd11e9 100644 --- a/crates/server-config/Cargo.toml +++ b/crates/server-config/Cargo.toml @@ -7,6 +7,7 @@ edition = "2021" [dependencies] config-utils = { workspace = true } fs-utils = { workspace = true } +cid-utils = { workspace = true } particle-protocol = { workspace = true } fluence-libp2p = { workspace = true, features = ["tokio"] } air-interpreter-fs = { workspace = true } @@ -35,6 +36,7 @@ bytesize = { version = "1.3.0", features = ["serde"] } serde_with = { workspace = true } config = { version = "0.13.4", default-features = false, features = ["toml"] } clarity = { workspace = true } +maplit = { workspace = true } [dev-dependencies] temp-env = "0.3.6" diff --git a/crates/server-config/src/defaults.rs b/crates/server-config/src/defaults.rs index f37a5072f6..f7e41c07e0 100644 --- a/crates/server-config/src/defaults.rs +++ b/crates/server-config/src/defaults.rs @@ -14,6 +14,7 @@ * limitations under the License. */ +use std::collections::HashMap; use std::net::IpAddr; use std::path::{Path, PathBuf}; use std::time::Duration; @@ -22,6 +23,7 @@ use libp2p::core::Multiaddr; use libp2p::identity::ed25519::Keypair; use libp2p::identity::PublicKey; use libp2p::PeerId; +use maplit::hashmap; use fluence_libp2p::Transport; @@ -102,6 +104,7 @@ pub fn default_base_dir() -> PathBuf { pub fn persistent_dir(base_dir: &Path) -> PathBuf { base_dir.join("persistent") } + pub fn ephemeral_dir(base_dir: &Path) -> PathBuf { base_dir.join("ephemeral") } @@ -257,3 +260,11 @@ pub fn default_decider_network_id() -> u64 { // 80001 = polygon mumbai 80001 } + +pub fn default_effectors() -> HashMap)> { + hashmap! { + "curl".to_string() => ("bafkreids22lgia5bqs63uigw4mqwhsoxvtnkpfqxqy5uwyyerrldsr32ce".to_string(), hashmap! { + "curl".to_string() => "/usr/bin/curl".to_string() + }) + } +} diff --git a/crates/server-config/src/node_config.rs b/crates/server-config/src/node_config.rs index 9743115b9d..8d0d8db56c 100644 --- a/crates/server-config/src/node_config.rs +++ b/crates/server-config/src/node_config.rs @@ -5,6 +5,7 @@ use std::path::{Path, PathBuf}; use std::time::Duration; use base64::{engine::general_purpose::STANDARD as base64, Engine}; +use cid_utils::Hash; use clarity::PrivateKey; use core_manager::CoreRange; use derivative::Derivative; @@ -130,9 +131,13 @@ pub struct UnresolvedNodeConfig { #[serde(default = "default_management_peer_id")] pub management_peer_id: PeerId, + // TODO: leave for now to migrate #[serde(default = "default_allowed_binaries")] pub allowed_binaries: Vec, + #[serde(default = "default_effectors_config")] + pub effectors: EffectorsConfig, + #[serde(default)] pub system_services: SystemServicesConfig, @@ -164,6 +169,17 @@ impl UnresolvedNodeConfig { allowed_binaries.push(self.system_services.aqua_ipfs.ipfs_binary_path.clone()); allowed_binaries.push(self.system_services.connector.curl_binary_path.clone()); + let allowed_effectors = self + .effectors + .0 + .into_values() + .map(|effector_config| { + let wasm_cid = effector_config.wasm_cid; + let allowed_binaries = effector_config.allowed_binaries; + (wasm_cid, allowed_binaries) + }) + .collect::<_>(); + let cpus_range = self.cpus_range.unwrap_or_default(); let result = NodeConfig { @@ -195,6 +211,7 @@ impl UnresolvedNodeConfig { transport_config: self.transport_config, listen_config: self.listen_config, allowed_binaries, + allowed_effectors, system_services: self.system_services, http_config: self.http_config, chain_config: self.chain_config, @@ -365,6 +382,8 @@ pub struct NodeConfig { pub allowed_binaries: Vec, + pub allowed_effectors: HashMap>, + pub system_services: SystemServicesConfig, pub http_config: Option, @@ -552,3 +571,35 @@ pub struct ChainListenerConfig { /// How often to poll proofs pub proof_poll_period: Duration, } + +/// Name of the effector module +/// Current is used only for users and is ignored by Nox +type EffectorModuleName = String; + +#[derive(Clone, Deserialize, Serialize, Derivative)] +#[derivative(Debug)] +pub struct EffectorsConfig(HashMap); + +#[derive(Clone, Deserialize, Serialize, Derivative)] +#[derivative(Debug)] +pub struct EffectorConfig { + #[derivative(Debug(format_with = "std::fmt::Display::fmt"))] + wasm_cid: Hash, + allowed_binaries: HashMap, +} + +fn default_effectors_config() -> EffectorsConfig { + let config = default_effectors() + .into_iter() + .map(|(module_name, config)| { + ( + module_name, + EffectorConfig { + wasm_cid: Hash::from_string(&config.0).unwrap(), + allowed_binaries: config.1, + }, + ) + }) + .collect::<_>(); + EffectorsConfig(config) +} diff --git a/crates/server-config/src/services_config.rs b/crates/server-config/src/services_config.rs index 77c685f9d4..0cd15deb45 100644 --- a/crates/server-config/src/services_config.rs +++ b/crates/server-config/src/services_config.rs @@ -17,8 +17,9 @@ use fs_utils::{create_dirs, set_write_only, to_abs_path}; use bytesize::ByteSize; +use cid_utils::Hash; use libp2p::PeerId; -use std::collections::{HashMap, HashSet}; +use std::collections::HashMap; use std::path::{Path, PathBuf}; #[derive(Debug, Clone)] @@ -47,8 +48,8 @@ pub struct ServicesConfig { pub builtins_management_peer_id: PeerId, /// Default heap size in bytes available for the module unless otherwise specified. pub default_service_memory_limit: Option, - /// List of allowed binaries paths - pub allowed_binaries: HashSet, + /// List of allowed effector modules by CID + pub allowed_effectors: HashMap>, } impl ServicesConfig { @@ -62,21 +63,27 @@ impl ServicesConfig { management_peer_id: PeerId, builtins_management_peer_id: PeerId, default_service_memory_limit: Option, - allowed_binaries: Vec, + allowed_effectors: HashMap>, ) -> Result { let persistent_dir = to_abs_path(persistent_dir); let ephemeral_dir = to_abs_path(ephemeral_dir); - let allowed_binaries = allowed_binaries + let allowed_effectors = allowed_effectors .into_iter() - .map(|path_str| { - let path = Path::new(&path_str); - match path.try_exists() { - Err(err) => log::warn!("cannot check binary `{path_str}`: {err}"), - Ok(false) => log::warn!("binary `{path_str}` does not exist"), - _ => {} - }; - path.to_path_buf() + .map(|(cid, effector)| { + let effector = effector + .into_iter() + .map(|(name, path_str)| { + let path = Path::new(&path_str); + match path.try_exists() { + Err(err) => log::warn!("cannot check effector `{path_str}`: {err}"), + Ok(false) => log::warn!("effector `{path_str}` does not exist"), + _ => {} + }; + (name, path.to_path_buf()) + }) + .collect::<_>(); + (cid, effector) }) .collect::<_>(); @@ -92,7 +99,7 @@ impl ServicesConfig { management_peer_id, builtins_management_peer_id, default_service_memory_limit, - allowed_binaries, + allowed_effectors, }; create_dirs(&[ diff --git a/crates/system-services/Cargo.toml b/crates/system-services/Cargo.toml index 30fc9aaec9..6a36844181 100644 --- a/crates/system-services/Cargo.toml +++ b/crates/system-services/Cargo.toml @@ -7,7 +7,7 @@ edition = "2021" [dependencies] aqua-ipfs-distro = "=0.6.0" -decider-distro = "=0.6.9" +decider-distro = "=0.6.10" registry-distro = "=0.9.4" trust-graph-distro = "=0.4.11" diff --git a/crates/system-services/src/deployer.rs b/crates/system-services/src/deployer.rs index 9d6102f6ef..a159b538b0 100644 --- a/crates/system-services/src/deployer.rs +++ b/crates/system-services/src/deployer.rs @@ -385,7 +385,7 @@ impl Deployer { )))?; let hash = self .modules_repo - .add_module(module.to_vec(), config) + .add_system_module(module.to_vec(), config) .map_err(|e| { eyre!( "error while adding module {name} of service `{}`: {:?}", diff --git a/nox/src/node.rs b/nox/src/node.rs index 13ffef654e..1e3fa944f1 100644 --- a/nox/src/node.rs +++ b/nox/src/node.rs @@ -158,7 +158,7 @@ impl Node { config.management_peer_id, builtins_peer_id, config.node_config.default_service_memory_limit, - config.node_config.allowed_binaries.clone(), + config.node_config.allowed_effectors.clone(), ) .expect("create services config"); @@ -237,12 +237,6 @@ impl Node { ) }; - let allowed_binaries = services_config - .allowed_binaries - .iter() - .map(|s| s.to_string_lossy().to_string()) - .collect::<_>(); - let mut builtins = Self::builtins( connectivity.clone(), services_config, @@ -307,11 +301,19 @@ impl Node { spell_metrics, ); + let allowed_binaries = config + .allowed_effectors + .values() + .flat_map(|v| v.values().cloned().collect::>()) + .collect::>() + .into_iter() + .collect::<_>(); let node_info = NodeInfo { external_addresses: config.external_addresses(), node_version: env!("CARGO_PKG_VERSION"), air_version: air_interpreter_wasm::VERSION, spell_version: spell_version.clone(), + // TODO: remove allowed_binaries, }; if let Some(m) = metrics_registry.as_mut() { @@ -580,8 +582,7 @@ impl Node { let chain_listener = self.chain_listener; task::Builder::new().name(&task_name.clone()).spawn(async move { - - let mut http_server = if let Some(http_listen_addr) = http_listen_addr{ + let mut http_server = if let Some(http_listen_addr) = http_listen_addr { tracing::info!("Starting http endpoint at {}", http_listen_addr); async move { start_http_endpoint(http_listen_addr, metrics_registry, health_registry, peer_id, versions, http_bind_outlet) diff --git a/particle-builtins/src/builtins.rs b/particle-builtins/src/builtins.rs index 250e65cb67..8e4adaf8dc 100644 --- a/particle-builtins/src/builtins.rs +++ b/particle-builtins/src/builtins.rs @@ -23,11 +23,12 @@ use std::sync::Arc; use std::time::{Duration, Instant}; use derivative::Derivative; +use fluence_app_service::TomlMarineNamedModuleConfig; use fluence_keypair::Signature; use libp2p::{core::Multiaddr, kad::KBucketKey, kad::K_VALUE, PeerId}; use multihash::Multihash; use serde::{Deserialize, Serialize}; -use serde_json::{json, Value as JValue}; +use serde_json::{json, Value as JValue, Value}; use tokio::sync::RwLock; use JValue::Array; @@ -108,7 +109,7 @@ where let modules_dir = &config.modules_dir; let blueprint_dir = &config.blueprint_dir; let modules = - ModuleRepository::new(modules_dir, blueprint_dir, config.allowed_binaries.clone()); + ModuleRepository::new(modules_dir, blueprint_dir, config.allowed_effectors.clone()); let services = ParticleAppServices::new( config, modules.clone(), @@ -205,6 +206,7 @@ where ("dist", "add_module_from_vault") => wrap(self.add_module_from_vault(args, particle)), ("dist", "add_module") => wrap(self.add_module(args)), + ("dist", "add_module_bytes_from_vault") => wrap(self.add_module_bytes_from_vault(args, particle)), ("dist", "add_blueprint") => wrap(self.add_blueprint(args)), ("dist", "make_module_config") => wrap(make_module_config(args)), ("dist", "load_module_config") => wrap(self.load_module_config_from_vault(args, particle)), @@ -283,7 +285,7 @@ where log::debug!(target: "run-console", "{}", json!(args.function_args)); } wrap_unit(Ok(())) - }, + } _ => FunctionOutcome::NotDefined { args, params: particle }, } @@ -617,13 +619,32 @@ where fn add_module_from_vault(&self, args: Args, params: ParticleParams) -> Result { let mut args = args.function_args.into_iter(); let module_path: String = Args::next("module_path", &mut args)?; - let config = Args::next("config", &mut args)?; + let config: TomlMarineNamedModuleConfig = Args::next("config", &mut args)?; let module_hash = self.modules.add_module_from_vault( &self.services.vault, self.scopes.to_peer_id(params.peer_scope), + config.name, + module_path, + params, + )?; + + Ok(json!(module_hash)) + } + + fn add_module_bytes_from_vault( + &self, + args: Args, + params: ParticleParams, + ) -> Result { + let mut args = args.function_args.into_iter(); + let module_name: String = Args::next("module_name", &mut args)?; + let module_path: String = Args::next("module_path", &mut args)?; + let module_hash = self.modules.add_module_from_vault( + &self.services.vault, + self.scopes.to_peer_id(params.peer_scope), + module_name, module_path, - config, params, )?; @@ -659,7 +680,6 @@ where Ok(config) } - fn default_module_config(&self, args: Args) -> Result { let mut args = args.function_args.into_iter(); let module_name: String = Args::next("module_name", &mut args)?; diff --git a/particle-modules/Cargo.toml b/particle-modules/Cargo.toml index 294d5439e3..823911a18c 100644 --- a/particle-modules/Cargo.toml +++ b/particle-modules/Cargo.toml @@ -13,6 +13,7 @@ fs-utils = { workspace = true } service-modules = { workspace = true } fluence-libp2p = { workspace = true } +marine-module-info-parser = { workspace = true } marine-it-parser = { workspace = true } fluence-app-service = { workspace = true } @@ -29,7 +30,9 @@ bytesize = "1.3.0" cid = { workspace = true } libipld = { workspace = true } itertools = { workspace = true } +maplit = { workspace = true } [dev-dependencies] tempdir = "0.3.7" server-config = { workspace = true } +maplit = { workspace = true } diff --git a/particle-modules/src/error.rs b/particle-modules/src/error.rs index 7bf4c4bf96..a3d90d8025 100644 --- a/particle-modules/src/error.rs +++ b/particle-modules/src/error.rs @@ -20,12 +20,13 @@ use std::path::PathBuf; use base64::DecodeError; use fluence_app_service::{MarineError, TomlMarineNamedModuleConfig}; use marine_it_parser::ITParserError; +use marine_module_info_parser::ModuleInfoError; use serde_json::Value as JValue; use thiserror::Error; use json_utils::err_as_value; use particle_execution::VaultError; -use service_modules::Blueprint; +use service_modules::{Blueprint, Hash}; pub(super) type Result = std::result::Result; @@ -145,10 +146,23 @@ pub enum ModuleError { max_heap_size_wanted: u64, max_heap_size_allowed: u64, }, - #[error("Config error: requested mounted binary {forbidden_path} is forbidden on this host")] - ForbiddenMountedBinary { forbidden_path: String }, + #[error("Config error: requested module effector {module_name} with CID {forbidden_cid} is forbidden on this host")] + ForbiddenEffector { + module_name: String, + forbidden_cid: Hash, + }, + #[error("Module {module_name} with CID {module_cid} requested a binary `{binary_name}` which isn't in the configured list of binaries for the effector")] + InvalidEffectorMountedBinary { + module_name: String, + module_cid: String, + binary_name: String, + }, #[error(transparent)] Vault(#[from] VaultError), + #[error(transparent)] + ModuleInfo(#[from] ModuleInfoError), + #[error(transparent)] + WrongModuleHash(#[from] eyre::ErrReport), } impl From for JValue { diff --git a/particle-modules/src/lib.rs b/particle-modules/src/lib.rs index a621f48fde..75ae66fe98 100644 --- a/particle-modules/src/lib.rs +++ b/particle-modules/src/lib.rs @@ -15,6 +15,7 @@ */ #![feature(try_blocks)] +#![feature(assert_matches)] #![warn(rust_2018_idioms)] #![deny( dead_code, diff --git a/particle-modules/src/modules.rs b/particle-modules/src/modules.rs index 5c8ad0eb53..d253e3778f 100644 --- a/particle-modules/src/modules.rs +++ b/particle-modules/src/modules.rs @@ -14,18 +14,16 @@ * limitations under the License. */ -use std::{ - collections::{HashMap, HashSet}, - path::Path, - path::PathBuf, - sync::Arc, -}; +use std::collections::HashSet; +use std::{collections::HashMap, path::Path, path::PathBuf, sync::Arc}; use base64::{engine::general_purpose::STANDARD as base64, Engine}; use eyre::WrapErr; -use fluence_app_service::{ModuleDescriptor, TomlMarineNamedModuleConfig}; +use fluence_app_service::{ModuleDescriptor, TomlMarineModuleConfig, TomlMarineNamedModuleConfig}; use fstrings::f; use marine_it_parser::module_interface; +use marine_module_info_parser::effects; +use marine_module_info_parser::effects::WasmEffect; use parking_lot::RwLock; use serde_json::{json, Value as JValue}; @@ -38,11 +36,14 @@ use service_modules::{ }; use crate::error::ModuleError::{ - BlueprintNotFound, EmptyDependenciesList, ForbiddenMountedBinary, ReadModuleInterfaceError, + BlueprintNotFound, EmptyDependenciesList, ReadModuleInterfaceError, }; use crate::error::Result; use crate::files::{self, load_config_by_path, load_module_descriptor}; -use crate::ModuleError::{IncorrectVaultModuleConfig, SerializeBlueprintJson}; +use crate::ModuleError::{ + ForbiddenEffector, IncorrectVaultModuleConfig, InvalidEffectorMountedBinary, + SerializeBlueprintJson, +}; #[derive(Debug, Clone)] pub struct ModuleRepository { @@ -50,14 +51,14 @@ pub struct ModuleRepository { blueprints_dir: PathBuf, module_interface_cache: Arc>>, blueprints: Arc>>, - allowed_binaries: HashSet, + allowed_effectors: HashMap>, } impl ModuleRepository { pub fn new( modules_dir: &Path, blueprints_dir: &Path, - allowed_binaries: HashSet, + allowed_effectors: HashMap>, ) -> Self { let blueprints = Self::load_blueprints(blueprints_dir); let blueprints_cache = Arc::new(RwLock::new(blueprints)); @@ -67,33 +68,53 @@ impl ModuleRepository { blueprints_dir: blueprints_dir.to_path_buf(), module_interface_cache: <_>::default(), blueprints: blueprints_cache, - allowed_binaries, + allowed_effectors, } } - fn check_module_mounted_binaries(&self, config: &TomlMarineNamedModuleConfig) -> Result<()> { - if let Some(binaries) = &config.config.mounted_binaries { - for requested_binary in binaries.values() { - if let Some(requested_binary) = requested_binary.as_str() { - let requested_binary_path = Path::new(requested_binary); - if !self.allowed_binaries.contains(requested_binary_path) { - return Err(ForbiddenMountedBinary { - forbidden_path: requested_binary.to_string(), + pub fn add_module(&self, name: String, module: Vec) -> Result { + // TODO: remove unwrap + let hash = Hash::new(&module)?; + let (logger_enabled, mounted) = Self::get_module_effects(&module)?; + let mut effector_settings = None; + if !mounted.is_empty() { + if let Some(binaries) = self.allowed_effectors.get(&hash) { + // check that all configured binaries are the same as in the module + // for now print warning if not + for binary_name in binaries.keys() { + if !mounted + .iter() + .any(|mounted_name| mounted_name == binary_name) + { + return Err(InvalidEffectorMountedBinary { + module_name: name, + module_cid: hash.to_string(), + binary_name: binary_name.clone(), }); } } + effector_settings = Some(binaries); + } else { + return Err(ForbiddenEffector { + module_name: name, + forbidden_cid: hash, + }); } - } - Ok(()) - } - - pub fn add_module(&self, module: Vec, config: TomlMarineNamedModuleConfig) -> Result { - // TODO: remove unwrap - let hash = Hash::new(&module).unwrap(); + }; + let config = Self::make_config(name, logger_enabled, effector_settings); + let _config = files::add_module(&self.modules_dir, &hash, &module, config)?; - let config = files::add_module(&self.modules_dir, &hash, &module, config)?; - self.check_module_mounted_binaries(&config)?; + Ok(hash) + } + // TODO: generate config for modules also + pub fn add_system_module( + &self, + module: Vec, + config: TomlMarineNamedModuleConfig, + ) -> Result { + let hash = Hash::new(&module)?; + let _config = files::add_module(&self.modules_dir, &hash, &module, config)?; Ok(hash) } @@ -116,7 +137,7 @@ impl ModuleRepository { config: TomlMarineNamedModuleConfig, ) -> Result { let module = base64.decode(module)?; - let hash = self.add_module(module, config)?; + let hash = self.add_module(config.name, module)?; Ok(hash.to_string()) } @@ -126,13 +147,13 @@ impl ModuleRepository { vault: &ParticleVault, // TODO: refactor this out of this crate current_peer_id: PeerId, + name: String, module_path: String, - config: TomlMarineNamedModuleConfig, particle: ParticleParams, ) -> Result { let module = vault.cat_slice(current_peer_id, &particle, Path::new(&module_path))?; // copy module & config to module_dir - let hash = self.add_module(module, config)?; + let hash = self.add_module(name, module)?; Ok(hash.to_string()) } @@ -172,7 +193,7 @@ impl ModuleRepository { let result = match result { Ok((hash, config)) => json!({ - "name": config.name, + "name": config.name, "hash": hash.to_string(), "config": config.config, }), @@ -306,6 +327,48 @@ impl ModuleRepository { Ok(module_descriptors) } + + fn get_module_effects(module: &[u8]) -> Result<(bool, HashSet)> { + let effects = effects::extract_from_bytes(module)?; + let mut logger_enabled = false; + let mut mounted_names = HashSet::new(); + for effect in effects { + match effect { + WasmEffect::Logger => { + logger_enabled = true; + } + WasmEffect::MountedBinary(name) => { + mounted_names.insert(name); + } + } + } + Ok((logger_enabled, mounted_names)) + } + + fn make_config( + module_name: String, + logger_enabled: bool, + effector_settings: Option<&HashMap>, + ) -> TomlMarineNamedModuleConfig { + let mounted_binaries = effector_settings.map(|effector_settings| { + effector_settings + .iter() + .map(|(name, path)| (name.clone(), path.to_string_lossy().to_string().into())) + .collect::<_>() + }); + + TomlMarineNamedModuleConfig { + name: module_name, + file_name: None, + load_from: None, + config: TomlMarineModuleConfig { + logger_enabled: Some(logger_enabled), + wasi: None, + mounted_binaries, + logging_mask: None, + }, + } + } } fn get_interface_by_hash( @@ -338,11 +401,16 @@ fn get_interface_by_hash( mod tests { use base64::{engine::general_purpose::STANDARD as base64, Engine}; use fluence_app_service::{TomlMarineModuleConfig, TomlMarineNamedModuleConfig}; + use maplit::hashmap; + use std::assert_matches::assert_matches; + use std::default::Default; + use std::path::PathBuf; use tempdir::TempDir; use service_modules::load_module; use service_modules::Hash; + use crate::ModuleError::{ForbiddenEffector, InvalidEffectorMountedBinary}; use crate::{AddBlueprint, ModuleRepository}; #[test] @@ -410,4 +478,95 @@ mod tests { let result = repo.get_interface(&m_hash); assert!(result.is_ok()) } + + #[test] + fn test_add_module_effector_allowed() { + let effector_wasm_cid = + Hash::from_string("bafkreiepzclggkt57vu7yrhxylfhaafmuogtqly7wel7ozl5k2ehkd44oe") + .unwrap(); + + let effector_path = "../crates/nox-tests/tests/effector/artifacts"; + let allowed_effectors = hashmap! { + effector_wasm_cid => hashmap! { + "ls".to_string() => PathBuf::from("/bin/ls"), + } + }; + + let module_dir = TempDir::new("test").unwrap(); + let bp_dir = TempDir::new("test2").unwrap(); + let repo = ModuleRepository::new(module_dir.path(), bp_dir.path(), allowed_effectors); + + let module = load_module(effector_path, "effector").expect("load module"); + let result = repo.add_module("effector".to_string(), module); + assert_matches!(result, Ok(_)); + } + + #[test] + fn test_add_module_effector_forbidden() { + let some_wasm_cid = + Hash::from_string("bafkreibjsugno2xsa2ee46xge5t6z4vuwpepyphedbykrfgmm7i6jg6ihe") + .unwrap(); + + let effector_path = "../crates/nox-tests/tests/effector/artifacts"; + let allowed_effectors = hashmap! { + some_wasm_cid => hashmap! { + "ls".to_string() => PathBuf::from("/bin/ls"), + "cat".to_string() => PathBuf::from("/bin/cat"), + } + }; + + let module_dir = TempDir::new("test").unwrap(); + let bp_dir = TempDir::new("test2").unwrap(); + let repo = ModuleRepository::new(module_dir.path(), bp_dir.path(), allowed_effectors); + + let module = load_module(effector_path, "effector").expect("load module"); + let result = repo.add_module("effector".to_string(), module); + assert_matches!(result, Err(ForbiddenEffector { .. })); + } + + #[test] + fn test_add_module_effector_invalid() { + let effector_wasm_cid = + Hash::from_string("bafkreiepzclggkt57vu7yrhxylfhaafmuogtqly7wel7ozl5k2ehkd44oe") + .unwrap(); + + let effector_path = "../crates/nox-tests/tests/effector/artifacts"; + let allowed_effectors = hashmap! { + effector_wasm_cid => hashmap! { + "ls".to_string() => PathBuf::from("/bin/ls"), + "cat".to_string() => PathBuf::from("/bin/cat"), + } + }; + + let module_dir = TempDir::new("test").unwrap(); + let bp_dir = TempDir::new("test2").unwrap(); + let repo = ModuleRepository::new(module_dir.path(), bp_dir.path(), allowed_effectors); + + let module = load_module(effector_path, "effector").expect("load module"); + let result = repo.add_module("effector".to_string(), module); + let _cat = "cat".to_string(); + assert_matches!( + result, + Err(InvalidEffectorMountedBinary { + binary_name: _cat, + .. + }) + ); + } + + #[test] + fn test_add_module_pure() { + let module_dir = TempDir::new("test").unwrap(); + let bp_dir = TempDir::new("test2").unwrap(); + let repo = ModuleRepository::new(module_dir.path(), bp_dir.path(), Default::default()); + + let module = load_module( + "../crates/nox-tests/tests/tetraplets/artifacts", + "tetraplets", + ) + .expect("load module"); + + let result = repo.add_module("pure".to_string(), module); + assert_matches!(result, Ok(_)); + } } diff --git a/spell-storage/src/storage.rs b/spell-storage/src/storage.rs index 79b425426b..d8be01b300 100644 --- a/spell-storage/src/storage.rs +++ b/spell-storage/src/storage.rs @@ -67,7 +67,7 @@ impl SpellStorage { config.name )))?; let module_hash = modules - .add_module(module.to_vec(), config) + .add_module(config.name, module.to_vec()) .context(format!("adding spell module {name}"))?; hashes.push(module_hash); } @@ -92,7 +92,7 @@ impl SpellStorage { .unwrap_or(PathBuf::from(module_file_name(&config.name))); let module_path = spells_base_dir.join(load_from); let module = load_module_by_path(module_path.as_ref())?; - let module_hash = modules.add_module(module, config)?; + let module_hash = modules.add_module(config.name, module)?; versions.push(String::from(&module_hash.to_string()[..8])); hashes.push(module_hash); }