feat: Implement Hermes IPFS runtime extension functionality (#272)

* feat(wit): Add bindings for hermes-ipfs

* feat(wit): add exports_hermes_ipfs_event_on_topic to stub-module.c

* feat(wip): add stubbed ipfs::api::Host implementation and on-topic event

* fix: update integration test modules

* feat: add ipfs runtime context

* fix: lints

* fix: update rust integration test modules

* chore: update project dictionary

* fix: pubsub-message includes topic

* fix: update IPFS WIT bindings

* fix: add errors and types, peer-evict function

* fix: typo

* chore: format code

* chore: add hermes-ipfs to deps

* chore: add docs to show how to start ipfs node

* feat: add methods to publish/get/pin files

* feat: add unsubscribe method, return friendlier types

* feat(wip): implement ipfs methods

* chore: format code and remove unused dep

* fix: add peer_evict method to Host implementation

* chore: format code

* fix: remove dup function

* feat: keep track of ipfs files belonging to apps

* send app name to hermes ipfs functions

* fix: remove unused import, update event.wit

* fix: remove unused import, update event.wit

* fix: update IPFS WIT bindings

* feat: implement IPFS api

* feat: add pubsub-publish to IPFS wasi definitions

* fix: cleanup WIT type definitions

* feat: add integration testing for IPFS runtime extension

* fix: update rte and add ipfs tests to earthly target

* fix: update hermes-ipfs example to handle content already pinned

* fix: remove re-exports

* fix: type_lenght_limit to build doctests

* chore: refactor 'mod task' out of state

* fix: types for pubsub-publishing

* fix: add publishing implementation

* chore: update docs

* wip: publish

* fix: cleanup code

* remove lint attributes
* DRY code
* improve docs

* fix: refactor ipfs api into its own module

* fix: simplify type usage

* fix: use new type for gossip message id

* fix: spelling

* fix: use HashSet instead of DashSet

* fix: broken integration tests

* fix: Update hermes/bin/src/runtime_extensions/hermes/ipfs/state/api.rs

Code cleanup

Co-authored-by: Apisit Ritreungroj <38898766+apskhem@users.noreply.github.com>

* chore: update TODO comments with issue url

* fix: Update wasm/integration-test/ipfs/src/lib.rs

Cleaner code

Co-authored-by: Apisit Ritreungroj <38898766+apskhem@users.noreply.github.com>

* chore: minor fix to trigger ci

* fix: earthly integration tests in Rust use --keep-ts flag

---------

Co-authored-by: Steven Johnson <stevenj@users.noreply.github.com>
Co-authored-by: Apisit Ritreungroj <38898766+apskhem@users.noreply.github.com>

.config/dictionaries/project.dic

@@ -72,6 +72,7 @@ genhtml
 GETFL
 getres
 gmtime
+gossipsub
 happ
 hardano
 hasher

hermes/Cargo.toml

@@ -58,6 +58,8 @@ pallas-hardano = { git = "https://github.com/input-output-hk/catalyst-pallas.git
 
 cardano-chain-follower = { path = "crates/cardano-chain-follower", version = "0.0.1" }
 
+hermes-ipfs = { path = "crates/hermes-ipfs", version = "0.0.1" }
+
 wasmtime = "20.0.2"
 rusty_ulid = "2.0.0"
 anyhow = "1.0.71"
@@ -103,5 +105,6 @@ ed25519-dalek = "2.1.1"
 x509-cert = "0.2.5"
 coset = "0.3.7"
 libipld = "0.16.0"
+libp2p = "0.53.2"
 rust-ipfs = "0.11.19"
 rustyline-async = "0.4.2"

hermes/Earthfile

@@ -80,6 +80,7 @@ test-wasm-integration:
     COPY ../wasm/integration-test/crypto+build/crypto.wasm ../wasm/test-components/
     COPY ../wasm/integration-test/cardano+build/cardano.wasm ../wasm/test-components/
     COPY ../wasm/integration-test/hashing+build/hashing.wasm ../wasm/test-components/
+    COPY ../wasm/integration-test/ipfs+build/ipfs.wasm ../wasm/test-components/
     COPY ../wasm/integration-test/localtime+build/localtime.wasm ../wasm/test-components/
     COPY ../wasm/integration-test/logger+build/logger.wasm ../wasm/test-components/
     COPY ../wasm/integration-test/sqlite+build/sqlite.wasm ../wasm/test-components/

hermes/bin/Cargo.toml

@@ -71,6 +71,7 @@ sha2 = { workspace = true }
 ed25519-dalek = { workspace = true, features = ["pem"] }
 x509-cert = { workspace = true, features = ["pem"] }
 coset = { workspace = true }
+hermes-ipfs = { workspace = true }
 
 [build-dependencies]
 build-info-build = { workspace = true }

hermes/bin/src/lib.rs

@@ -1,5 +1,6 @@
 //! Intentionally empty
 //! This file exists, so that doc tests can be used inside binary crates.
+#![type_length_limit = "45079293105"]
 
 pub mod app;
 #[allow(dead_code)]

hermes/bin/src/runtime_extensions/hermes/ipfs/event.rs

@@ -1,15 +1,12 @@
 //! Hermes IPFS runtime extension event handler implementation.
 use crate::{
-    event::HermesEventPayload,
-    runtime_extensions::bindings::hermes::ipfs::api::{PubsubMessage, PubsubTopic},
+    event::HermesEventPayload, runtime_extensions::bindings::hermes::ipfs::api::PubsubMessage,
 };
 
 /// Event handler for the `on-topic` event.
-#[allow(dead_code)]
+#[derive(Debug, Clone)]
 pub(crate) struct OnTopicEvent {
-    ///  Topic
-    pub(crate) topic: PubsubTopic,
-    ///  Message
+    ///  Topic message received.
     pub(crate) message: PubsubMessage,
 }
 

hermes/bin/src/runtime_extensions/hermes/ipfs/host.rs

@@ -1,40 +1,58 @@
 //! IPFS host implementation for WASM runtime.
 
+use super::state::{
+    hermes_ipfs_add_file, hermes_ipfs_content_validate, hermes_ipfs_evict_peer,
+    hermes_ipfs_get_dht_value, hermes_ipfs_get_file, hermes_ipfs_pin_file, hermes_ipfs_publish,
+    hermes_ipfs_put_dht_value, hermes_ipfs_subscribe,
+};
 use crate::{
     runtime_context::HermesRuntimeContext,
     runtime_extensions::bindings::hermes::ipfs::api::{
-        DhtKey, DhtValue, Errno, Host, IpfsContent, IpfsPath, PeerId, PubsubTopic,
+        DhtKey, DhtValue, Errno, Host, IpfsContent, IpfsFile, IpfsPath, MessageData, MessageId,
+        PeerId, PubsubTopic,
     },
 };
 
 impl Host for HermesRuntimeContext {
-    fn file_add(&mut self, _contents: IpfsContent) -> wasmtime::Result<Result<IpfsPath, Errno>> {
-        todo!();
+    fn file_add(&mut self, contents: IpfsFile) -> wasmtime::Result<Result<IpfsPath, Errno>> {
+        let path: IpfsPath = hermes_ipfs_add_file(self.app_name(), contents)?.to_string();
+        Ok(Ok(path))
     }
 
-    fn file_get(&mut self, _path: IpfsPath) -> wasmtime::Result<Result<IpfsContent, Errno>> {
-        todo!();
+    fn file_get(&mut self, path: IpfsPath) -> wasmtime::Result<Result<IpfsFile, Errno>> {
+        let contents = hermes_ipfs_get_file(self.app_name(), &path)?;
+        Ok(Ok(contents))
     }
 
-    fn file_pin(&mut self, _ipfs_path: IpfsPath) -> wasmtime::Result<Result<bool, Errno>> {
-        todo!();
+    fn file_pin(&mut self, ipfs_path: IpfsPath) -> wasmtime::Result<Result<bool, Errno>> {
+        Ok(hermes_ipfs_pin_file(self.app_name(), ipfs_path))
     }
 
-    fn dht_put(
-        &mut self, _key: DhtKey, _contents: IpfsContent,
-    ) -> wasmtime::Result<Result<bool, Errno>> {
-        todo!();
+    fn dht_put(&mut self, key: DhtKey, value: DhtValue) -> wasmtime::Result<Result<bool, Errno>> {
+        Ok(hermes_ipfs_put_dht_value(self.app_name(), key, value))
+    }
+
+    fn dht_get(&mut self, key: DhtKey) -> wasmtime::Result<Result<DhtValue, Errno>> {
+        Ok(hermes_ipfs_get_dht_value(self.app_name(), key))
     }
 
-    fn dht_get(&mut self, _key: DhtKey) -> wasmtime::Result<Result<DhtValue, Errno>> {
-        todo!();
+    fn pubsub_publish(
+        &mut self, topic: PubsubTopic, message: MessageData,
+    ) -> wasmtime::Result<Result<MessageId, Errno>> {
+        Ok(hermes_ipfs_publish(self.app_name(), &topic, message))
     }
 
-    fn pubsub_subscribe(&mut self, _topic: PubsubTopic) -> wasmtime::Result<Result<bool, Errno>> {
-        todo!();
+    fn pubsub_subscribe(&mut self, topic: PubsubTopic) -> wasmtime::Result<Result<bool, Errno>> {
+        Ok(hermes_ipfs_subscribe(self.app_name(), topic))
+    }
+
+    fn ipfs_content_validate(
+        &mut self, content: IpfsContent,
+    ) -> wasmtime::Result<Result<bool, Errno>> {
+        Ok(Ok(hermes_ipfs_content_validate(self.app_name(), &content)))
     }
 
-    fn peer_evict(&mut self, _peer: PeerId) -> wasmtime::Result<Result<bool, Errno>> {
-        todo!();
+    fn peer_evict(&mut self, peer: PeerId) -> wasmtime::Result<Result<bool, Errno>> {
+        Ok(hermes_ipfs_evict_peer(self.app_name(), peer))
     }
 }

hermes/bin/src/runtime_extensions/hermes/ipfs/mod.rs

@@ -1,6 +1,7 @@
 //! Hermes IPFS runtime extension.
 mod event;
 mod host;
+mod state;
 
 /// Advise Runtime Extensions of a new context
 pub(crate) fn new_context(_ctx: &crate::runtime_context::HermesRuntimeContext) {}

hermes/bin/src/runtime_extensions/hermes/ipfs/state/api.rs

@@ -0,0 +1,125 @@
+//! Hermes IPFS State API
+use super::{is_valid_dht_content, is_valid_pubsub_content, HERMES_IPFS_STATE};
+use crate::{
+    app::HermesAppName,
+    runtime_extensions::bindings::hermes::ipfs::api::{
+        DhtKey, DhtValue, Errno, IpfsContent, IpfsFile, IpfsPath, MessageData, MessageId, PeerId,
+        PubsubTopic,
+    },
+};
+
+/// Add File to IPFS
+pub(crate) fn hermes_ipfs_add_file(
+    app_name: &HermesAppName, contents: IpfsFile,
+) -> Result<IpfsPath, Errno> {
+    tracing::debug!(app_name = %app_name, "adding IPFS file");
+    let ipfs_path = HERMES_IPFS_STATE.file_add(contents)?;
+    tracing::debug!(app_name = %app_name, path = %ipfs_path, "added IPFS file");
+    HERMES_IPFS_STATE
+        .apps
+        .added_file(app_name.clone(), ipfs_path.clone());
+    Ok(ipfs_path)
+}
+
+/// Validate IPFS Content from DHT or `PubSub`
+pub(crate) fn hermes_ipfs_content_validate(
+    app_name: &HermesAppName, content: &IpfsContent,
+) -> bool {
+    match content {
+        IpfsContent::Dht((k, v)) => {
+            let key_str = format!("{k:x?}");
+            let is_valid = is_valid_dht_content(k, v);
+            tracing::debug!(app_name = %app_name, dht_key = %key_str, is_valid = %is_valid, "DHT value validation");
+            is_valid
+        },
+        IpfsContent::Pubsub((topic, message)) => {
+            let is_valid = is_valid_pubsub_content(topic, message);
+            tracing::debug!(app_name = %app_name, topic = %topic, is_valid = %is_valid, "PubSub message validation");
+            is_valid
+        },
+    }
+}
+
+/// Get File from Ipfs
+pub(crate) fn hermes_ipfs_get_file(
+    app_name: &HermesAppName, path: &IpfsPath,
+) -> Result<IpfsFile, Errno> {
+    tracing::debug!(app_name = %app_name, path = %path, "get IPFS file");
+    let content = HERMES_IPFS_STATE.file_get(path)?;
+    tracing::debug!(app_name = %app_name, path = %path, "got IPFS file");
+    Ok(content)
+}
+
+/// Pin IPFS File
+pub(crate) fn hermes_ipfs_pin_file(
+    app_name: &HermesAppName, path: IpfsPath,
+) -> Result<bool, Errno> {
+    tracing::debug!(app_name = %app_name, path = %path, "pin IPFS file");
+    let status = HERMES_IPFS_STATE.file_pin(&path)?;
+    tracing::debug!(app_name = %app_name, path = %path, "pinned IPFS file");
+    HERMES_IPFS_STATE.apps.pinned_file(app_name.clone(), path);
+    Ok(status)
+}
+
+/// Get DHT Value
+pub(crate) fn hermes_ipfs_get_dht_value(
+    app_name: &HermesAppName, key: DhtKey,
+) -> Result<DhtValue, Errno> {
+    let key_str = format!("{key:x?}");
+    tracing::debug!(app_name = %app_name, dht_key = %key_str, "get DHT value");
+    let value = HERMES_IPFS_STATE.dht_get(key)?;
+    tracing::debug!(app_name = %app_name, dht_key = %key_str, "got DHT value");
+    Ok(value)
+}
+
+/// Put DHT Value
+pub(crate) fn hermes_ipfs_put_dht_value(
+    app_name: &HermesAppName, key: DhtKey, value: DhtValue,
+) -> Result<bool, Errno> {
+    let key_str = format!("{key:x?}");
+    tracing::debug!(app_name = %app_name, dht_key = %key_str, "putting DHT value");
+    let status = HERMES_IPFS_STATE.dht_put(key.clone(), value)?;
+    tracing::debug!(app_name = %app_name, dht_key = %key_str, "have put DHT value");
+    HERMES_IPFS_STATE.apps.added_dht_key(app_name.clone(), key);
+    Ok(status)
+}
+
+/// Subscribe to a topic
+pub(crate) fn hermes_ipfs_subscribe(
+    app_name: &HermesAppName, topic: PubsubTopic,
+) -> Result<bool, Errno> {
+    tracing::debug!(app_name = %app_name, pubsub_topic = %topic, "subscribing to PubSub topic");
+    if HERMES_IPFS_STATE.apps.topic_subscriptions_contains(&topic) {
+        tracing::debug!(app_name = %app_name, pubsub_topic = %topic, "topic subscription stream already exists");
+    } else {
+        let handle = HERMES_IPFS_STATE.pubsub_subscribe(&topic)?;
+        HERMES_IPFS_STATE
+            .apps
+            .added_topic_stream(topic.clone(), handle);
+        tracing::debug!(app_name = %app_name, pubsub_topic = %topic, "added subscription topic stream");
+    }
+    HERMES_IPFS_STATE
+        .apps
+        .added_app_topic_subscription(app_name.clone(), topic);
+    Ok(true)
+}
+
+/// Publish message to a topic
+pub(crate) fn hermes_ipfs_publish(
+    _app_name: &HermesAppName, topic: &PubsubTopic, message: MessageData,
+) -> Result<MessageId, Errno> {
+    HERMES_IPFS_STATE
+        .pubsub_publish(topic.to_string(), message)
+        .map(|m| m.0 .0)
+}
+
+/// Evict Peer from node
+pub(crate) fn hermes_ipfs_evict_peer(
+    app_name: &HermesAppName, peer: PeerId,
+) -> Result<bool, Errno> {
+    tracing::debug!(app_name = %app_name, peer_id = %peer, "evicting peer");
+    let status = HERMES_IPFS_STATE.peer_evict(&peer.to_string())?;
+    tracing::debug!(app_name = %app_name, peer_id = %peer, "evicted peer");
+    HERMES_IPFS_STATE.apps.evicted_peer(app_name.clone(), peer);
+    Ok(status)
+}