diff --git a/Cargo.lock b/Cargo.lock index 068599f828..4e112089b2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5573,6 +5573,7 @@ name = "particle-execution" version = "0.1.0" dependencies = [ "async-trait", + "bs58", "fluence-app-service", "fluence-libp2p", "fs-utils", diff --git a/crates/spell-service-api/src/lib.rs b/crates/spell-service-api/src/lib.rs index e9ea57a9ce..4193227b20 100644 --- a/crates/spell-service-api/src/lib.rs +++ b/crates/spell-service-api/src/lib.rs @@ -347,7 +347,6 @@ mod tests { let repo = ModuleRepository::new( &config.modules_dir, &config.blueprint_dir, - &config.particles_vault_dir, Default::default(), ); diff --git a/particle-builtins/src/builtins.rs b/particle-builtins/src/builtins.rs index 42abb04b68..a8aaa91c6d 100644 --- a/particle-builtins/src/builtins.rs +++ b/particle-builtins/src/builtins.rs @@ -17,7 +17,6 @@ use std::collections::{HashMap, HashSet}; use std::fmt::Debug; use std::ops::Try; -use std::path; use std::path::Path; use std::str::FromStr; use std::sync::Arc; @@ -85,8 +84,6 @@ pub struct Builtins { #[derivative(Debug(format_with = "fmt_custom_services"))] pub custom_services: RwLock>, - particles_vault_dir: path::PathBuf, - #[derivative(Debug = "ignore")] key_storage: Arc, #[derivative(Debug = "ignore")] @@ -110,14 +107,8 @@ where ) -> Self { let modules_dir = &config.modules_dir; let blueprint_dir = &config.blueprint_dir; - let vault_dir = &config.particles_vault_dir; - let modules = ModuleRepository::new( - modules_dir, - blueprint_dir, - vault_dir, - config.allowed_binaries.clone(), - ); - let particles_vault_dir = vault_dir.to_path_buf(); + let modules = + ModuleRepository::new(modules_dir, blueprint_dir, config.allowed_binaries.clone()); let services = ParticleAppServices::new( config, modules.clone(), @@ -131,7 +122,6 @@ where connectivity, modules, services, - particles_vault_dir, custom_services: <_>::default(), key_storage, scopes: scope, @@ -629,9 +619,12 @@ where let module_path: String = Args::next("module_path", &mut args)?; let config = Args::next("config", &mut args)?; - let module_hash = self - .modules - .add_module_from_vault(module_path, config, params)?; + let module_hash = self.modules.add_module_from_vault( + &self.services.vault, + module_path, + config, + params, + )?; Ok(json!(module_hash)) } @@ -654,9 +647,11 @@ where let mut args = args.function_args.into_iter(); let config_path: String = Args::next("config_path", &mut args)?; - let config = self - .modules - .load_module_config_from_vault(config_path, params)?; + let config = ModuleRepository::load_module_config_from_vault( + &self.services.vault, + config_path, + params, + )?; let config = serde_json::to_value(config) .map_err(|err| JError::new(format!("Error serializing config to JSON: {err}")))?; @@ -697,11 +692,17 @@ where params: ParticleParams, ) -> Result { let mut args = args.function_args.into_iter(); - let blueprint_path = Args::next("blueprint_path", &mut args)?; + let blueprint_path: String = Args::next("blueprint_path", &mut args)?; - let blueprint = self - .modules - .load_blueprint_from_vault(blueprint_path, params)?; + let data = self + .services + .vault + .cat_slice(¶ms, Path::new(&blueprint_path))?; + let blueprint = AddBlueprint::decode(&data).map_err(|err| { + JError::new(format!( + "Error parsing blueprint from vault {blueprint_path:?}: {err}" + )) + })?; let blueprint = blueprint .to_string() @@ -985,10 +986,7 @@ where let mut args = args.function_args.into_iter(); let data: String = Args::next("data", &mut args)?; let name = uuid(); - let virtual_path = self - .services - .vault - .put(¶ms.id, Path::new(&name), &data)?; + let virtual_path = self.services.vault.put(¶ms, name, &data)?; Ok(JValue::String(virtual_path.display().to_string())) } @@ -998,7 +996,7 @@ where let path: String = Args::next("path", &mut args)?; self.services .vault - .cat(¶ms.id, Path::new(&path)) + .cat(¶ms, Path::new(&path)) .map(JValue::String) .map_err(|_| JError::new(format!("Error reading vault file `{path}`"))) } diff --git a/particle-execution/Cargo.toml b/particle-execution/Cargo.toml index 859fccfe9f..420012f5a9 100644 --- a/particle-execution/Cargo.toml +++ b/particle-execution/Cargo.toml @@ -16,6 +16,7 @@ fluence-app-service = { workspace = true } thiserror = { workspace = true } futures = { workspace = true } serde_json = { workspace = true } +bs58 = { workspace = true } tokio = { workspace = true, features = ["fs"] } parking_lot = { workspace = true } diff --git a/particle-execution/src/particle_vault.rs b/particle-execution/src/particle_vault.rs index 2370c62440..253895a355 100644 --- a/particle-execution/src/particle_vault.rs +++ b/particle-execution/src/particle_vault.rs @@ -23,6 +23,7 @@ use thiserror::Error; use fs_utils::create_dir; +use crate::ParticleParams; use crate::VaultError::WrongVault; use VaultError::{CleanupVault, CreateVault, InitializeVault}; @@ -30,7 +31,7 @@ pub const VIRTUAL_PARTICLE_VAULT_PREFIX: &str = "/tmp/vault"; #[derive(Debug, Clone)] pub struct ParticleVault { - pub vault_dir: PathBuf, + vault_dir: PathBuf, } impl ParticleVault { @@ -38,8 +39,14 @@ impl ParticleVault { Self { vault_dir } } - pub fn particle_vault(&self, key: &str) -> PathBuf { - self.vault_dir.join(key) + /// Returns Particle File Vault path on Nox's filesystem + pub fn particle_vault(&self, particle_id: &str) -> PathBuf { + self.vault_dir.join(particle_id) + } + + /// Returns Particle File Vault path on Marine's filesystem (ie how it would look like inside service) + pub fn virtual_particle_vault(&self, particle_id: &str) -> PathBuf { + Path::new(VIRTUAL_PARTICLE_VAULT_PREFIX).join(particle_id) } pub async fn initialize(&self) -> Result<(), VaultError> { @@ -50,33 +57,44 @@ impl ParticleVault { Ok(()) } - pub fn create(&self, particle_id: &str) -> Result<(), VaultError> { - let path = self.particle_vault(particle_id); + pub fn create(&self, particle: &ParticleParams) -> Result<(), VaultError> { + let path = self.particle_vault(&particle.id); create_dir(path).map_err(CreateVault)?; Ok(()) } + pub fn exists(&self, particle: &ParticleParams) -> bool { + self.particle_vault(&particle.id).exists() + } + pub fn put( &self, - particle_id: &str, - path: &Path, + particle: &ParticleParams, + filename: String, payload: &str, ) -> Result { - let vault_dir = self.particle_vault(particle_id); - let real_path = vault_dir.join(path); + let vault_dir = self.particle_vault(&particle.id); + // Note that we can't use `to_real_path` here since the target file cannot exist yet, + // but `to_real_path` do path normalization which requires existence of the file to resolve + // symlinks. + let real_path = vault_dir.join(&filename); if let Some(parent_path) = real_path.parent() { create_dir(parent_path).map_err(CreateVault)?; } std::fs::write(real_path.clone(), payload.as_bytes()) - .map_err(|e| VaultError::WriteVault(e, path.to_path_buf()))?; + .map_err(|e| VaultError::WriteVault(e, filename))?; - self.to_virtual_path(&real_path, particle_id) + self.to_virtual_path(&real_path, &particle.id) } - pub fn cat(&self, particle_id: &str, virtual_path: &Path) -> Result { - let real_path = self.to_real_path(virtual_path, particle_id)?; + pub fn cat( + &self, + particle: &ParticleParams, + virtual_path: &Path, + ) -> Result { + let real_path = self.to_real_path(virtual_path, &particle.id)?; let contents = std::fs::read_to_string(real_path) .map_err(|e| VaultError::ReadVault(e, virtual_path.to_path_buf()))?; @@ -84,6 +102,15 @@ impl ParticleVault { Ok(contents) } + pub fn cat_slice( + &self, + particle: &ParticleParams, + virtual_path: &Path, + ) -> Result, VaultError> { + let real_path = self.to_real_path(virtual_path, &particle.id)?; + std::fs::read(real_path).map_err(|e| VaultError::ReadVault(e, virtual_path.to_path_buf())) + } + pub async fn cleanup(&self, particle_id: &str) -> Result<(), VaultError> { let path = self.particle_vault(particle_id); match tokio::fs::remove_dir_all(&path).await { @@ -99,8 +126,8 @@ impl ParticleVault { /// Converts real path in `vault_dir` to virtual path with `VIRTUAL_PARTICLE_VAULT_PREFIX`. /// Virtual path looks like `/tmp/vault//`. fn to_virtual_path(&self, path: &Path, particle_id: &str) -> Result { - let virtual_prefix = path::Path::new(VIRTUAL_PARTICLE_VAULT_PREFIX).join(particle_id); - let real_prefix = self.vault_dir.join(particle_id); + let virtual_prefix = self.virtual_particle_vault(particle_id); + let real_prefix = self.particle_vault(particle_id); let rest = path .strip_prefix(&real_prefix) .map_err(|e| WrongVault(Some(e), path.to_path_buf(), real_prefix))?; @@ -109,13 +136,20 @@ impl ParticleVault { } /// Converts virtual path with `VIRTUAL_PARTICLE_VAULT_PREFIX` to real path in `vault_dir`. + /// Support full paths to the file in the vault starting this the prefix as well as relative paths + /// inside the vault. + /// For example, `some/file.txt` is valid and will be resolved to `REAL_PARTICLE_VAULT_PREFIX/some/file.txt`. fn to_real_path(&self, path: &Path, particle_id: &str) -> Result { - let virtual_prefix = path::Path::new(VIRTUAL_PARTICLE_VAULT_PREFIX).join(particle_id); - let real_prefix = self.vault_dir.join(particle_id); - - let rest = path - .strip_prefix(&virtual_prefix) - .map_err(|e| WrongVault(Some(e), path.to_path_buf(), virtual_prefix.clone()))?; + let rest = if path.has_root() { + // If path starts with the `/` then we consider it a full path containing the virtual vault prefix + let virtual_prefix = self.virtual_particle_vault(particle_id); + path.strip_prefix(&virtual_prefix) + .map_err(|e| WrongVault(Some(e), path.to_path_buf(), virtual_prefix.clone()))? + } else { + // Otherwise we consider it a relative path inside the vault + path + }; + let real_prefix = self.particle_vault(particle_id); let real_path = real_prefix.join(rest); let resolved_path = real_path .canonicalize() @@ -148,11 +182,11 @@ impl ParticleVault { #[derive(Debug, Error)] pub enum VaultError { - #[error("error creating vault_dir")] + #[error("Error creating vault_dir")] InitializeVault(#[source] std::io::Error), - #[error("error creating particle vault")] + #[error("Error creating particle vault")] CreateVault(#[source] std::io::Error), - #[error("error cleaning up particle vault")] + #[error("Error cleaning up particle vault")] CleanupVault(#[source] std::io::Error), #[error("Incorrect vault path `{1}`: doesn't belong to vault (`{2}`)")] WrongVault(#[source] Option, PathBuf, PathBuf), @@ -160,6 +194,6 @@ pub enum VaultError { NotFound(#[source] std::io::Error, PathBuf), #[error("Read vault failed for `{1}`: {0}")] ReadVault(#[source] std::io::Error, PathBuf), - #[error("Write vault failed for `{1}`: {0}")] - WriteVault(#[source] std::io::Error, PathBuf), + #[error("Write vault failed for filename `{1}`: {0}")] + WriteVault(#[source] std::io::Error, String), } diff --git a/particle-modules/src/error.rs b/particle-modules/src/error.rs index a1ed61d177..7bf4c4bf96 100644 --- a/particle-modules/src/error.rs +++ b/particle-modules/src/error.rs @@ -24,6 +24,7 @@ use serde_json::Value as JValue; use thiserror::Error; use json_utils::err_as_value; +use particle_execution::VaultError; use service_modules::Blueprint; pub(super) type Result = std::result::Result; @@ -131,45 +132,12 @@ pub enum ModuleError { #[source] err: eyre::Report, }, - #[error("Vault directory for this particle doesn't exist. You must call a service first.")] - VaultDoesNotExist { vault_path: PathBuf }, - #[error("Module not found in vault at {module_path:?}")] - ModuleNotFoundInVault { - module_path: PathBuf, - #[source] - err: std::io::Error, - }, - #[error("Config not found in vault at {config_path:?}")] - ConfigNotFoundInVault { - config_path: PathBuf, - #[source] - err: std::io::Error, - }, #[error("Error parsing module config from vault {config_path:?}: {err}")] IncorrectVaultModuleConfig { - config_path: PathBuf, + config_path: String, #[source] err: serde_json::Error, }, - #[error("Invalid blueprint path {blueprint_path:?}: {err}")] - InvalidBlueprintPath { - blueprint_path: String, - #[source] - err: eyre::Report, - }, - #[error("Blueprint not found in vault at {blueprint_path:?}")] - BlueprintNotFoundInVault { - blueprint_path: PathBuf, - #[source] - err: std::io::Error, - }, - #[error("Error parsing blueprint from vault {blueprint_path:?}: {err}")] - IncorrectVaultBlueprint { - blueprint_path: PathBuf, - #[source] - err: eyre::Report, - }, - #[error( "Config error: max_heap_size = '{max_heap_size_wanted}' can't be bigger than {max_heap_size_allowed}'" )] @@ -179,6 +147,8 @@ pub enum ModuleError { }, #[error("Config error: requested mounted binary {forbidden_path} is forbidden on this host")] ForbiddenMountedBinary { forbidden_path: String }, + #[error(transparent)] + Vault(#[from] VaultError), } impl From for JValue { diff --git a/particle-modules/src/modules.rs b/particle-modules/src/modules.rs index 9eb83f8b16..ed0dbb445b 100644 --- a/particle-modules/src/modules.rs +++ b/particle-modules/src/modules.rs @@ -29,29 +29,24 @@ use marine_it_parser::module_interface; use parking_lot::RwLock; use serde_json::{json, Value as JValue}; -use fs_utils::file_name; use particle_args::JError; -use particle_execution::ParticleParams; +use particle_execution::{ParticleParams, ParticleVault}; use service_modules::{ extract_module_file_name, is_blueprint, module_config_name_hash, module_file_name_hash, AddBlueprint, Blueprint, Hash, }; use crate::error::ModuleError::{ - BlueprintNotFound, BlueprintNotFoundInVault, ConfigNotFoundInVault, EmptyDependenciesList, - ForbiddenMountedBinary, IncorrectVaultBlueprint, IncorrectVaultModuleConfig, - InvalidBlueprintPath, InvalidModuleConfigPath, InvalidModulePath, ModuleNotFoundInVault, - ReadModuleInterfaceError, VaultDoesNotExist, + BlueprintNotFound, EmptyDependenciesList, ForbiddenMountedBinary, ReadModuleInterfaceError, }; use crate::error::Result; use crate::files::{self, load_config_by_path, load_module_descriptor}; -use crate::ModuleError::SerializeBlueprintJson; +use crate::ModuleError::{IncorrectVaultModuleConfig, SerializeBlueprintJson}; #[derive(Debug, Clone)] pub struct ModuleRepository { modules_dir: PathBuf, blueprints_dir: PathBuf, - particles_vault_dir: PathBuf, module_interface_cache: Arc>>, blueprints: Arc>>, allowed_binaries: HashSet, @@ -61,7 +56,6 @@ impl ModuleRepository { pub fn new( modules_dir: &Path, blueprints_dir: &Path, - particles_vault_dir: &Path, allowed_binaries: HashSet, ) -> Self { let blueprints = Self::load_blueprints(blueprints_dir); @@ -72,7 +66,6 @@ impl ModuleRepository { blueprints_dir: blueprints_dir.to_path_buf(), module_interface_cache: <_>::default(), blueprints: blueprints_cache, - particles_vault_dir: particles_vault_dir.to_path_buf(), allowed_binaries, } } @@ -103,60 +96,16 @@ impl ModuleRepository { Ok(hash) } - fn check_vault_exists(&self, particle_id: &str) -> Result { - let vault_path = self.particles_vault_dir.join(particle_id); - if !vault_path.exists() { - return Err(VaultDoesNotExist { vault_path }); - } - Ok(vault_path) - } - pub fn load_module_config_from_vault( - &self, + vault: &ParticleVault, config_path: String, particle: ParticleParams, ) -> Result { - let vault_path = self.check_vault_exists(&particle.id)?; - // load & deserialize module config from vault - let config_fname = - file_name(&config_path).map_err(|err| InvalidModuleConfigPath { err, config_path })?; - let config_path = vault_path.join(config_fname); - let config = std::fs::read(&config_path).map_err(|err| { - let config_path = config_path.clone(); - ConfigNotFoundInVault { config_path, err } - })?; - + let config = vault.cat_slice(&particle, Path::new(&config_path))?; serde_json::from_slice(&config) .map_err(|err| IncorrectVaultModuleConfig { config_path, err }) } - pub fn load_blueprint_from_vault( - &self, - blueprint_path: String, - particle: ParticleParams, - ) -> Result { - let vault_path = self.check_vault_exists(&particle.id)?; - - // load & deserialize module config from vault - let blueprint_fname = file_name(&blueprint_path).map_err(|err| InvalidBlueprintPath { - err, - blueprint_path, - })?; - let blueprint_path = vault_path.join(blueprint_fname); - let data = std::fs::read(&blueprint_path).map_err(|err| { - let blueprint_path = blueprint_path.clone(); - BlueprintNotFoundInVault { - blueprint_path, - err, - } - })?; - - AddBlueprint::decode(&data).map_err(|err| IncorrectVaultBlueprint { - blueprint_path, - err, - }) - } - /// Adds a module to the filesystem, overwriting existing module. pub fn add_module_base64( &self, @@ -171,19 +120,12 @@ impl ModuleRepository { pub fn add_module_from_vault( &self, + vault: &ParticleVault, module_path: String, config: TomlMarineNamedModuleConfig, particle: ParticleParams, ) -> Result { - let vault_path = self.check_vault_exists(&particle.id)?; - - // load module - let module_fname = - file_name(&module_path).map_err(|err| InvalidModulePath { err, module_path })?; - let module_path = vault_path.join(module_fname); - let module = std::fs::read(&module_path) - .map_err(|err| ModuleNotFoundInVault { module_path, err })?; - + let module = vault.cat_slice(&particle, Path::new(&module_path))?; // copy module & config to module_dir let hash = self.add_module(module, config)?; @@ -402,13 +344,7 @@ mod tests { fn test_add_blueprint() { let module_dir = TempDir::new("test").unwrap(); let bp_dir = TempDir::new("test").unwrap(); - let vault_dir = TempDir::new("test").unwrap(); - let repo = ModuleRepository::new( - module_dir.path(), - bp_dir.path(), - vault_dir.path(), - Default::default(), - ); + let repo = ModuleRepository::new(module_dir.path(), bp_dir.path(), Default::default()); let dep1 = Hash::new(&[1, 2, 3]).unwrap(); let dep2 = Hash::new(&[3, 2, 1]).unwrap(); @@ -442,13 +378,7 @@ mod tests { fn test_add_module_get_interface() { let module_dir = TempDir::new("test").unwrap(); let bp_dir = TempDir::new("test2").unwrap(); - let vault_dir = TempDir::new("test3").unwrap(); - let repo = ModuleRepository::new( - module_dir.path(), - bp_dir.path(), - vault_dir.path(), - Default::default(), - ); + let repo = ModuleRepository::new(module_dir.path(), bp_dir.path(), Default::default()); let module = load_module( "../crates/nox-tests/tests/tetraplets/artifacts", diff --git a/particle-services/src/app_services.rs b/particle-services/src/app_services.rs index 773ceb3e72..f5f7187b5c 100644 --- a/particle-services/src/app_services.rs +++ b/particle-services/src/app_services.rs @@ -431,7 +431,7 @@ impl ParticleAppServices { // TODO: move particle vault creation to aquamarine::particle_functions if create_vault { - self.vault.create(&particle.id)?; + self.vault.create(&particle)?; } let call_parameters_worker_id = self.scopes.to_peer_id(peer_scope); @@ -1135,7 +1135,6 @@ mod tests { let repo = ModuleRepository::new( &config.modules_dir, &config.blueprint_dir, - &config.particles_vault_dir, Default::default(), );