fix(particle-vault): refactor particle vault usage (#2077)

* use signature in the vault virtual and real paths

* fix test build

* revert signature usage

* fix

crates/spell-service-api/src/lib.rs

@@ -347,7 +347,6 @@ mod tests {
         let repo = ModuleRepository::new(
             &config.modules_dir,
             &config.blueprint_dir,
-            &config.particles_vault_dir,
             Default::default(),
         );
 

particle-builtins/src/builtins.rs

@@ -17,7 +17,6 @@
 use std::collections::{HashMap, HashSet};
 use std::fmt::Debug;
 use std::ops::Try;
-use std::path;
 use std::path::Path;
 use std::str::FromStr;
 use std::sync::Arc;
@@ -85,8 +84,6 @@ pub struct Builtins<C> {
     #[derivative(Debug(format_with = "fmt_custom_services"))]
     pub custom_services: RwLock<HashMap<String, CustomService>>,
 
-    particles_vault_dir: path::PathBuf,
-
     #[derivative(Debug = "ignore")]
     key_storage: Arc<KeyStorage>,
     #[derivative(Debug = "ignore")]
@@ -110,14 +107,8 @@ where
     ) -> Self {
         let modules_dir = &config.modules_dir;
         let blueprint_dir = &config.blueprint_dir;
-        let vault_dir = &config.particles_vault_dir;
-        let modules = ModuleRepository::new(
-            modules_dir,
-            blueprint_dir,
-            vault_dir,
-            config.allowed_binaries.clone(),
-        );
-        let particles_vault_dir = vault_dir.to_path_buf();
+        let modules =
+            ModuleRepository::new(modules_dir, blueprint_dir, config.allowed_binaries.clone());
         let services = ParticleAppServices::new(
             config,
             modules.clone(),
@@ -131,7 +122,6 @@ where
             connectivity,
             modules,
             services,
-            particles_vault_dir,
             custom_services: <_>::default(),
             key_storage,
             scopes: scope,
@@ -629,9 +619,12 @@ where
         let module_path: String = Args::next("module_path", &mut args)?;
         let config = Args::next("config", &mut args)?;
 
-        let module_hash = self
-            .modules
-            .add_module_from_vault(module_path, config, params)?;
+        let module_hash = self.modules.add_module_from_vault(
+            &self.services.vault,
+            module_path,
+            config,
+            params,
+        )?;
 
         Ok(json!(module_hash))
     }
@@ -654,9 +647,11 @@ where
         let mut args = args.function_args.into_iter();
         let config_path: String = Args::next("config_path", &mut args)?;
 
-        let config = self
-            .modules
-            .load_module_config_from_vault(config_path, params)?;
+        let config = ModuleRepository::load_module_config_from_vault(
+            &self.services.vault,
+            config_path,
+            params,
+        )?;
         let config = serde_json::to_value(config)
             .map_err(|err| JError::new(format!("Error serializing config to JSON: {err}")))?;
 
@@ -697,11 +692,17 @@ where
         params: ParticleParams,
     ) -> Result<JValue, JError> {
         let mut args = args.function_args.into_iter();
-        let blueprint_path = Args::next("blueprint_path", &mut args)?;
+        let blueprint_path: String = Args::next("blueprint_path", &mut args)?;
 
-        let blueprint = self
-            .modules
-            .load_blueprint_from_vault(blueprint_path, params)?;
+        let data = self
+            .services
+            .vault
+            .cat_slice(&params, Path::new(&blueprint_path))?;
+        let blueprint = AddBlueprint::decode(&data).map_err(|err| {
+            JError::new(format!(
+                "Error parsing blueprint from vault {blueprint_path:?}: {err}"
+            ))
+        })?;
 
         let blueprint = blueprint
             .to_string()
@@ -985,10 +986,7 @@ where
         let mut args = args.function_args.into_iter();
         let data: String = Args::next("data", &mut args)?;
         let name = uuid();
-        let virtual_path = self
-            .services
-            .vault
-            .put(&params.id, Path::new(&name), &data)?;
+        let virtual_path = self.services.vault.put(&params, name, &data)?;
 
         Ok(JValue::String(virtual_path.display().to_string()))
     }
@@ -998,7 +996,7 @@ where
         let path: String = Args::next("path", &mut args)?;
         self.services
             .vault
-            .cat(&params.id, Path::new(&path))
+            .cat(&params, Path::new(&path))
             .map(JValue::String)
             .map_err(|_| JError::new(format!("Error reading vault file `{path}`")))
     }

particle-execution/Cargo.toml

@@ -16,6 +16,7 @@ fluence-app-service = { workspace = true }
 thiserror = { workspace = true }
 futures = { workspace = true }
 serde_json = { workspace = true }
+bs58 = { workspace = true }
 
 tokio = { workspace = true, features = ["fs"] }
 parking_lot = { workspace = true }

particle-execution/src/particle_vault.rs

@@ -23,23 +23,30 @@ use thiserror::Error;
 
 use fs_utils::create_dir;
 
+use crate::ParticleParams;
 use crate::VaultError::WrongVault;
 use VaultError::{CleanupVault, CreateVault, InitializeVault};
 
 pub const VIRTUAL_PARTICLE_VAULT_PREFIX: &str = "/tmp/vault";
 
 #[derive(Debug, Clone)]
 pub struct ParticleVault {
-    pub vault_dir: PathBuf,
+    vault_dir: PathBuf,
 }
 
 impl ParticleVault {
     pub fn new(vault_dir: PathBuf) -> Self {
         Self { vault_dir }
     }
 
-    pub fn particle_vault(&self, key: &str) -> PathBuf {
-        self.vault_dir.join(key)
+    /// Returns Particle File Vault path on Nox's filesystem
+    pub fn particle_vault(&self, particle_id: &str) -> PathBuf {
+        self.vault_dir.join(particle_id)
+    }
+
+    /// Returns Particle File Vault path on Marine's filesystem (ie how it would look like inside service)
+    pub fn virtual_particle_vault(&self, particle_id: &str) -> PathBuf {
+        Path::new(VIRTUAL_PARTICLE_VAULT_PREFIX).join(particle_id)
     }
 
     pub async fn initialize(&self) -> Result<(), VaultError> {
@@ -50,40 +57,60 @@ impl ParticleVault {
         Ok(())
     }
 
-    pub fn create(&self, particle_id: &str) -> Result<(), VaultError> {
-        let path = self.particle_vault(particle_id);
+    pub fn create(&self, particle: &ParticleParams) -> Result<(), VaultError> {
+        let path = self.particle_vault(&particle.id);
         create_dir(path).map_err(CreateVault)?;
 
         Ok(())
     }
 
+    pub fn exists(&self, particle: &ParticleParams) -> bool {
+        self.particle_vault(&particle.id).exists()
+    }
+
     pub fn put(
         &self,
-        particle_id: &str,
-        path: &Path,
+        particle: &ParticleParams,
+        filename: String,
         payload: &str,
     ) -> Result<PathBuf, VaultError> {
-        let vault_dir = self.particle_vault(particle_id);
-        let real_path = vault_dir.join(path);
+        let vault_dir = self.particle_vault(&particle.id);
+        // Note that we can't use `to_real_path` here since the target file cannot exist yet,
+        // but `to_real_path` do path normalization which requires existence of the file to resolve
+        // symlinks.
+        let real_path = vault_dir.join(&filename);
         if let Some(parent_path) = real_path.parent() {
             create_dir(parent_path).map_err(CreateVault)?;
         }
 
         std::fs::write(real_path.clone(), payload.as_bytes())
-            .map_err(|e| VaultError::WriteVault(e, path.to_path_buf()))?;
+            .map_err(|e| VaultError::WriteVault(e, filename))?;
 
-        self.to_virtual_path(&real_path, particle_id)
+        self.to_virtual_path(&real_path, &particle.id)
     }
 
-    pub fn cat(&self, particle_id: &str, virtual_path: &Path) -> Result<String, VaultError> {
-        let real_path = self.to_real_path(virtual_path, particle_id)?;
+    pub fn cat(
+        &self,
+        particle: &ParticleParams,
+        virtual_path: &Path,
+    ) -> Result<String, VaultError> {
+        let real_path = self.to_real_path(virtual_path, &particle.id)?;
 
         let contents = std::fs::read_to_string(real_path)
             .map_err(|e| VaultError::ReadVault(e, virtual_path.to_path_buf()))?;
 
         Ok(contents)
     }
 
+    pub fn cat_slice(
+        &self,
+        particle: &ParticleParams,
+        virtual_path: &Path,
+    ) -> Result<Vec<u8>, VaultError> {
+        let real_path = self.to_real_path(virtual_path, &particle.id)?;
+        std::fs::read(real_path).map_err(|e| VaultError::ReadVault(e, virtual_path.to_path_buf()))
+    }
+
     pub async fn cleanup(&self, particle_id: &str) -> Result<(), VaultError> {
         let path = self.particle_vault(particle_id);
         match tokio::fs::remove_dir_all(&path).await {
@@ -99,8 +126,8 @@ impl ParticleVault {
     /// Converts real path in `vault_dir` to virtual path with `VIRTUAL_PARTICLE_VAULT_PREFIX`.
     /// Virtual path looks like `/tmp/vault/<particle_id>/<path>`.
     fn to_virtual_path(&self, path: &Path, particle_id: &str) -> Result<PathBuf, VaultError> {
-        let virtual_prefix = path::Path::new(VIRTUAL_PARTICLE_VAULT_PREFIX).join(particle_id);
-        let real_prefix = self.vault_dir.join(particle_id);
+        let virtual_prefix = self.virtual_particle_vault(particle_id);
+        let real_prefix = self.particle_vault(particle_id);
         let rest = path
             .strip_prefix(&real_prefix)
             .map_err(|e| WrongVault(Some(e), path.to_path_buf(), real_prefix))?;
@@ -109,13 +136,20 @@ impl ParticleVault {
     }
 
     /// Converts virtual path with `VIRTUAL_PARTICLE_VAULT_PREFIX` to real path in `vault_dir`.
+    /// Support full paths to the file in the vault starting this the prefix as well as relative paths
+    /// inside the vault.
+    /// For example, `some/file.txt` is valid and will be resolved to `REAL_PARTICLE_VAULT_PREFIX/some/file.txt`.
     fn to_real_path(&self, path: &Path, particle_id: &str) -> Result<PathBuf, VaultError> {
-        let virtual_prefix = path::Path::new(VIRTUAL_PARTICLE_VAULT_PREFIX).join(particle_id);
-        let real_prefix = self.vault_dir.join(particle_id);
-
-        let rest = path
-            .strip_prefix(&virtual_prefix)
-            .map_err(|e| WrongVault(Some(e), path.to_path_buf(), virtual_prefix.clone()))?;
+        let rest = if path.has_root() {
+            // If path starts with the `/` then we consider it a full path containing the virtual vault prefix
+            let virtual_prefix = self.virtual_particle_vault(particle_id);
+            path.strip_prefix(&virtual_prefix)
+                .map_err(|e| WrongVault(Some(e), path.to_path_buf(), virtual_prefix.clone()))?
+        } else {
+            // Otherwise we consider it a relative path inside the vault
+            path
+        };
+        let real_prefix = self.particle_vault(particle_id);
         let real_path = real_prefix.join(rest);
         let resolved_path = real_path
             .canonicalize()
@@ -148,18 +182,18 @@ impl ParticleVault {
 
 #[derive(Debug, Error)]
 pub enum VaultError {
-    #[error("error creating vault_dir")]
+    #[error("Error creating vault_dir")]
     InitializeVault(#[source] std::io::Error),
-    #[error("error creating particle vault")]
+    #[error("Error creating particle vault")]
     CreateVault(#[source] std::io::Error),
-    #[error("error cleaning up particle vault")]
+    #[error("Error cleaning up particle vault")]
     CleanupVault(#[source] std::io::Error),
     #[error("Incorrect vault path `{1}`: doesn't belong to vault (`{2}`)")]
     WrongVault(#[source] Option<path::StripPrefixError>, PathBuf, PathBuf),
     #[error("Incorrect vault  path `{1}`: doesn't exist")]
     NotFound(#[source] std::io::Error, PathBuf),
     #[error("Read vault failed for `{1}`: {0}")]
     ReadVault(#[source] std::io::Error, PathBuf),
-    #[error("Write vault failed for `{1}`: {0}")]
-    WriteVault(#[source] std::io::Error, PathBuf),
+    #[error("Write vault failed for filename `{1}`: {0}")]
+    WriteVault(#[source] std::io::Error, String),
 }

particle-modules/src/error.rs

@@ -24,6 +24,7 @@ use serde_json::Value as JValue;
 use thiserror::Error;
 
 use json_utils::err_as_value;
+use particle_execution::VaultError;
 use service_modules::Blueprint;
 
 pub(super) type Result<T> = std::result::Result<T, ModuleError>;
@@ -131,45 +132,12 @@ pub enum ModuleError {
         #[source]
         err: eyre::Report,
     },
-    #[error("Vault directory for this particle doesn't exist. You must call a service first.")]
-    VaultDoesNotExist { vault_path: PathBuf },
-    #[error("Module not found in vault at {module_path:?}")]
-    ModuleNotFoundInVault {
-        module_path: PathBuf,
-        #[source]
-        err: std::io::Error,
-    },
-    #[error("Config not found in vault at {config_path:?}")]
-    ConfigNotFoundInVault {
-        config_path: PathBuf,
-        #[source]
-        err: std::io::Error,
-    },
     #[error("Error parsing module config from vault {config_path:?}: {err}")]
     IncorrectVaultModuleConfig {
-        config_path: PathBuf,
+        config_path: String,
         #[source]
         err: serde_json::Error,
     },
-    #[error("Invalid blueprint path {blueprint_path:?}: {err}")]
-    InvalidBlueprintPath {
-        blueprint_path: String,
-        #[source]
-        err: eyre::Report,
-    },
-    #[error("Blueprint not found in vault at {blueprint_path:?}")]
-    BlueprintNotFoundInVault {
-        blueprint_path: PathBuf,
-        #[source]
-        err: std::io::Error,
-    },
-    #[error("Error parsing blueprint from vault {blueprint_path:?}: {err}")]
-    IncorrectVaultBlueprint {
-        blueprint_path: PathBuf,
-        #[source]
-        err: eyre::Report,
-    },
-
     #[error(
     "Config error: max_heap_size = '{max_heap_size_wanted}' can't be bigger than {max_heap_size_allowed}'"
     )]
@@ -179,6 +147,8 @@ pub enum ModuleError {
     },
     #[error("Config error: requested mounted binary {forbidden_path} is forbidden on this host")]
     ForbiddenMountedBinary { forbidden_path: String },
+    #[error(transparent)]
+    Vault(#[from] VaultError),
 }
 
 impl From<ModuleError> for JValue {