feat(builtins): add vault.cat, vault.put + refactoring [NET-489 NET-491…

…] (#1631)

crates/nox-tests/tests/script_storage.rs

@@ -602,7 +602,7 @@ async fn add_script_from_vault_wrong_vault() {
     ).await.unwrap();
 
     if let [JValue::String(error_msg)] = result.as_slice() {
-        let expected_error_prefix = "Local service error, ret_code is 1, error message is '\"Error: Incorrect vault path `/tmp/vault/another-particle-id/script";
+        let expected_error_prefix = r#"Local service error, ret_code is 1, error message is '"Error reading script file `/tmp/vault/another-particle-id/script`: Incorrect vault path"#;
         assert!(
             error_msg.starts_with(expected_error_prefix),
             "expected:\n{expected_error_prefix}\ngot:\n{error_msg}"

crates/nox-tests/tests/vault.rs

@@ -217,3 +217,40 @@ async fn load_blueprint_from_vault() {
         panic!("#incorrect args: expected a single string, got {:?}", args);
     }
 }
+
+#[tokio::test]
+async fn put_cat_vault() {
+    let swarms = make_swarms(1).await;
+
+    let mut client = ConnectedClient::connect_to(swarms[0].multiaddr.clone())
+        .await
+        .wrap_err("connect client")
+        .unwrap();
+
+    let payload = "test-test-test".to_string();
+
+    client.send_particle(
+        r#"
+        (seq
+            (seq
+                (call relay ("vault" "put") [payload] filename)
+                (call relay ("vault" "cat") [filename] output_content)
+            )
+            (call %init_peer_id% ("op" "return") [output_content])
+        )
+        "#,
+        hashmap! {
+            "relay" => json!(client.node.to_string()),
+            "payload" => json!(payload.clone()),
+        },
+    );
+
+    use serde_json::Value::String;
+
+    let args = client.receive_args().await.unwrap();
+    if let [String(output)] = args.as_slice() {
+        assert_eq!(*output, payload);
+    } else {
+        panic!("incorrect args: expected a single string, got {:?}", args);
+    }
+}

particle-builtins/src/builtins.rs

@@ -19,6 +19,7 @@ use std::collections::{HashMap, HashSet};
 use std::fmt::Debug;
 use std::ops::Try;
 use std::path;
+use std::path::Path;
 use std::str::FromStr;
 use std::time::{Duration, Instant};
 
@@ -42,10 +43,11 @@ use particle_modules::{
     AddBlueprint, ModuleConfig, ModuleRepository, NamedModuleConfig, WASIConfig,
 };
 use particle_protocol::Contact;
-use particle_services::{ParticleAppServices, ServiceType, VIRTUAL_PARTICLE_VAULT_PREFIX};
+use particle_services::{ParticleAppServices, ServiceType};
 use peer_metrics::ServicesMetrics;
 use script_storage::ScriptStorageApi;
 use server_config::ServicesConfig;
+use uuid_utils::uuid;
 
 use crate::debug::fmt_custom_services;
 use crate::error::HostClosureCallError;
@@ -284,6 +286,8 @@ where
             ("json", "obj_pairs") => unary(args, |vs: Vec<(String, JValue)>| -> R<JValue, _> { json::obj_from_pairs(vs) }),
             ("json", "puts_pairs") => binary(args, |obj: JValue, vs: Vec<(String, JValue)>| -> R<JValue, _> { json::puts_from_pairs(obj, vs) }),
 
+            ("vault", "put") => wrap(self.vault_put(args, particle)),
+            ("vault", "cat") => wrap(self.vault_cat(args, particle)),
             ("run-console", "print")          => wrap_unit(Ok(log::debug!(target: "run-console", "{}", json!(args.function_args)))),
 
             _ => FunctionOutcome::NotDefined { args, params: particle },
@@ -413,9 +417,13 @@ where
         path: &path::Path,
         particle_id: &str,
     ) -> Result<String, JError> {
-        let resolved_path = resolve_vault_path(&self.particles_vault_dir, path, particle_id)?;
-        std::fs::read_to_string(resolved_path)
-            .map_err(|_| JError::new(format!("Error reading script file `{}`", path.display())))
+        self.services.vault.cat(particle_id, path).map_err(|e| {
+            JError::new(format!(
+                "Error reading script file `{}`: {}",
+                path.display(),
+                e
+            ))
+        })
     }
 
     async fn remove_script(&self, args: Args, params: ParticleParams) -> Result<JValue, JError> {
@@ -1048,6 +1056,28 @@ where
             self.key_manager.insecure_keypair.get_peer_id().to_base58(),
         ))
     }
+
+    fn vault_put(&self, args: Args, params: ParticleParams) -> Result<JValue, JError> {
+        let mut args = args.function_args.into_iter();
+        let data: String = Args::next("data", &mut args)?;
+        let name = uuid();
+        let virtual_path = self
+            .services
+            .vault
+            .put(&params.id, Path::new(&name), &data)?;
+
+        Ok(JValue::String(virtual_path.display().to_string()))
+    }
+
+    fn vault_cat(&self, args: Args, params: ParticleParams) -> Result<JValue, JError> {
+        let mut args = args.function_args.into_iter();
+        let path: String = Args::next("path", &mut args)?;
+        self.services
+            .vault
+            .cat(&params.id, Path::new(&path))
+            .map(JValue::String)
+            .map_err(|_| JError::new(format!("Error reading vault file `{path}`")))
+    }
 }
 
 fn make_module_config(args: Args) -> Result<JValue, JError> {
@@ -1151,46 +1181,6 @@ fn get_delay(delay: Option<Duration>, interval: Option<Duration>) -> Duration {
     }
 }
 
-#[derive(thiserror::Error, Debug)]
-enum ResolveVaultError {
-    #[error("Incorrect vault path `{1}`: doesn't belong to vault (`{2}`)")]
-    WrongVault(
-        #[source] Option<path::StripPrefixError>,
-        path::PathBuf,
-        path::PathBuf,
-    ),
-    #[error("Incorrect vault  path `{1}`: doesn't exist")]
-    NotFound(#[source] std::io::Error, path::PathBuf),
-}
-
-/// Map the given virtual path to the real one from the file system of the node.
-fn resolve_vault_path(
-    particles_vault_dir: &path::Path,
-    path: &path::Path,
-    particle_id: &str,
-) -> Result<path::PathBuf, ResolveVaultError> {
-    let virtual_prefix = path::Path::new(VIRTUAL_PARTICLE_VAULT_PREFIX).join(particle_id);
-    let real_prefix = particles_vault_dir.join(particle_id);
-
-    let rest = path.strip_prefix(&virtual_prefix).map_err(|e| {
-        ResolveVaultError::WrongVault(Some(e), path.to_path_buf(), virtual_prefix.clone())
-    })?;
-    let real_path = real_prefix.join(rest);
-    let resolved_path = real_path
-        .canonicalize()
-        .map_err(|e| ResolveVaultError::NotFound(e, path.to_path_buf()))?;
-    // Check again after normalization that the path leads to the real particle vault
-    if resolved_path.starts_with(&real_prefix) {
-        Ok(resolved_path)
-    } else {
-        Err(ResolveVaultError::WrongVault(
-            None,
-            resolved_path,
-            real_prefix,
-        ))
-    }
-}
-
 #[cfg(test)]
 mod prop_tests {
     use std::str::FromStr;
@@ -1273,75 +1263,3 @@ mod prop_tests {
         }
     }
 }
-
-#[cfg(test)]
-mod resolve_path_tests {
-    use std::fs::File;
-    use std::path::Path;
-
-    use particle_services::VIRTUAL_PARTICLE_VAULT_PREFIX;
-
-    use crate::builtins::{resolve_vault_path, ResolveVaultError};
-
-    fn with_env(callback: fn(&str, &Path, &str, &Path) -> ()) {
-        let particle_id = "particle_id";
-        let dir = tempfile::tempdir().expect("can't create temp dir");
-        let real_vault_prefix = dir.path().canonicalize().expect("").join("vault");
-        let real_vault_dir = real_vault_prefix.join(particle_id);
-        std::fs::create_dir_all(&real_vault_dir).expect("can't create dirs");
-
-        let filename = "file";
-        let real_path = real_vault_dir.join(filename);
-        File::create(&real_path).expect("can't create a file");
-
-        callback(
-            particle_id,
-            real_vault_prefix.as_path(),
-            filename,
-            real_path.as_path(),
-        );
-
-        dir.close().ok();
-    }
-
-    #[test]
-    fn test_resolve_path_ok() {
-        with_env(|particle_id, real_prefix, filename, path| {
-            let virtual_path = Path::new(VIRTUAL_PARTICLE_VAULT_PREFIX)
-                .join(particle_id)
-                .join(filename);
-            let result = resolve_vault_path(real_prefix, &virtual_path, particle_id).unwrap();
-            assert_eq!(result, path);
-        });
-    }
-
-    #[test]
-    fn test_resolve_path_wrong_vault() {
-        with_env(|particle_id, real_prefix, filename, _path| {
-            let virtual_path = Path::new(VIRTUAL_PARTICLE_VAULT_PREFIX)
-                .join("other-particle-id")
-                .join(filename);
-            let result = resolve_vault_path(real_prefix, &virtual_path, particle_id);
-            assert!(result.is_err());
-            assert!(matches!(
-                result.unwrap_err(),
-                ResolveVaultError::WrongVault(_, _, _)
-            ));
-        });
-    }
-
-    #[test]
-    fn test_resolve_path_not_found() {
-        with_env(|particle_id, real_prefix, _filename, _path| {
-            let virtual_path = Path::new(VIRTUAL_PARTICLE_VAULT_PREFIX)
-                .join(particle_id)
-                .join("other-file");
-            let result = resolve_vault_path(real_prefix, &virtual_path, particle_id);
-            assert!(result.is_err());
-            assert!(matches!(
-                result.unwrap_err(),
-                ResolveVaultError::NotFound(_, _)
-            ));
-        });
-    }
-}

particle-execution/Cargo.toml

@@ -10,6 +10,7 @@ particle-args = { workspace = true }
 fluence-libp2p = { workspace = true }
 fs-utils = { workspace = true }
 json-utils = { workspace = true }
+fluence-app-service = { workspace = true }
 
 thiserror = { workspace = true }
 futures = { workspace = true }

particle-execution/src/lib.rs

@@ -7,7 +7,7 @@ pub use particle_function::{
     ParticleFunctionStatic, ServiceFunction, ServiceFunctionImmut, ServiceFunctionMut,
 };
 pub use particle_params::ParticleParams;
-pub use particle_vault::{ParticleVault, VaultError};
+pub use particle_vault::{ParticleVault, VaultError, VIRTUAL_PARTICLE_VAULT_PREFIX};
 
 mod function_outcome;
 mod particle_function;

particle-execution/src/particle_vault.rs

@@ -14,14 +14,19 @@
  * limitations under the License.
  */
 
-use std::path::PathBuf;
+use fluence_app_service::{MarineWASIConfig, ModuleDescriptor};
+use std::path;
+use std::path::{Path, PathBuf};
 
 use thiserror::Error;
 
 use fs_utils::{create_dir, remove_dir};
 
+use crate::VaultError::WrongVault;
 use VaultError::{CleanupVault, CreateVault, InitializeVault};
 
+pub const VIRTUAL_PARTICLE_VAULT_PREFIX: &str = "/tmp/vault";
+
 #[derive(Debug, Clone)]
 pub struct ParticleVault {
     pub vault_dir: PathBuf,
@@ -49,11 +54,87 @@ impl ParticleVault {
         Ok(())
     }
 
+    pub fn put(
+        &self,
+        particle_id: &str,
+        path: &Path,
+        payload: &str,
+    ) -> Result<PathBuf, VaultError> {
+        let vault_dir = self.particle_vault(particle_id);
+        let real_path = vault_dir.join(path);
+        if let Some(parent_path) = real_path.parent() {
+            create_dir(parent_path).map_err(CreateVault)?;
+        }
+
+        std::fs::write(real_path.clone(), payload.as_bytes())
+            .map_err(|e| VaultError::WriteVault(e, path.to_path_buf()))?;
+
+        self.to_virtual_path(&real_path, particle_id)
+    }
+
+    pub fn cat(&self, particle_id: &str, virtual_path: &Path) -> Result<String, VaultError> {
+        let real_path = self.to_real_path(virtual_path, particle_id)?;
+
+        let contents = std::fs::read_to_string(real_path)
+            .map_err(|e| VaultError::ReadVault(e, virtual_path.to_path_buf()))?;
+
+        Ok(contents)
+    }
+
     pub fn cleanup(&self, particle_id: &str) -> Result<(), VaultError> {
         remove_dir(&self.particle_vault(particle_id)).map_err(CleanupVault)?;
 
         Ok(())
     }
+
+    /// Converts real path in `vault_dir` to virtual path with `VIRTUAL_PARTICLE_VAULT_PREFIX`.
+    /// Virtual path looks like `/tmp/vault/<particle_id>/<path>`.
+    fn to_virtual_path(&self, path: &Path, particle_id: &str) -> Result<PathBuf, VaultError> {
+        let virtual_prefix = path::Path::new(VIRTUAL_PARTICLE_VAULT_PREFIX).join(particle_id);
+        let real_prefix = self.vault_dir.join(particle_id);
+        let rest = path
+            .strip_prefix(&real_prefix)
+            .map_err(|e| WrongVault(Some(e), path.to_path_buf(), real_prefix))?;
+
+        Ok(virtual_prefix.join(rest))
+    }
+
+    /// Converts virtual path with `VIRTUAL_PARTICLE_VAULT_PREFIX` to real path in `vault_dir`.
+    fn to_real_path(&self, path: &Path, particle_id: &str) -> Result<PathBuf, VaultError> {
+        let virtual_prefix = path::Path::new(VIRTUAL_PARTICLE_VAULT_PREFIX).join(particle_id);
+        let real_prefix = self.vault_dir.join(particle_id);
+
+        let rest = path
+            .strip_prefix(&virtual_prefix)
+            .map_err(|e| WrongVault(Some(e), path.to_path_buf(), virtual_prefix.clone()))?;
+        let real_path = real_prefix.join(rest);
+        let resolved_path = real_path
+            .canonicalize()
+            .map_err(|e| VaultError::NotFound(e, path.to_path_buf()))?;
+        // Check again after normalization that the path leads to the real particle vault
+        if resolved_path.starts_with(&real_prefix) {
+            Ok(resolved_path)
+        } else {
+            Err(WrongVault(None, resolved_path, real_prefix))
+        }
+    }
+
+    /// Map `vault_dir` to `/tmp/vault` inside the service.
+    /// Particle File Vaults will be available as `/tmp/vault/$particle_id`
+    pub fn inject_vault(&self, module: &mut ModuleDescriptor) {
+        let wasi = &mut module.config.wasi;
+        if wasi.is_none() {
+            *wasi = Some(MarineWASIConfig::default());
+        }
+        // SAFETY: set wasi to Some in the code above
+        let wasi = wasi.as_mut().unwrap();
+
+        let vault_dir = self.vault_dir.to_path_buf();
+
+        wasi.preopened_files.insert(vault_dir.clone());
+        wasi.mapped_dirs
+            .insert(VIRTUAL_PARTICLE_VAULT_PREFIX.into(), vault_dir);
+    }
 }
 
 #[derive(Debug, Error)]
@@ -64,4 +145,12 @@ pub enum VaultError {
     CreateVault(#[source] std::io::Error),
     #[error("error cleaning up particle vault")]
     CleanupVault(#[source] std::io::Error),
+    #[error("Incorrect vault path `{1}`: doesn't belong to vault (`{2}`)")]
+    WrongVault(#[source] Option<path::StripPrefixError>, PathBuf, PathBuf),
+    #[error("Incorrect vault  path `{1}`: doesn't exist")]
+    NotFound(#[source] std::io::Error, PathBuf),
+    #[error("Read vault failed for `{1}`: {0}")]
+    ReadVault(#[source] std::io::Error, PathBuf),
+    #[error("Write vault failed for `{1}`: {0}")]
+    WriteVault(#[source] std::io::Error, PathBuf),
 }