Store thesaurus using persistence trait

crates/config/src/lib.rs

@@ -218,7 +218,8 @@ impl Default for Config {
 
 #[async_trait]
 impl Persistable for Config {
-    fn new() -> Self {
+    fn new(_key: String) -> Self {
+        // Key is not used because we use the `id` field
         Config::new()
     }
 

crates/middleware/Cargo.toml

@@ -7,6 +7,7 @@ edition = "2021"
 terraphim_config = { path = "../config" }
 terraphim_rolegraph = { path = "../terraphim_rolegraph" }
 terraphim_types = { path = "../../terraphim_types" }
+persistence = { path = "../persistence" }
 
 ahash = { version = "0.8.8", features = ["serde"] }
 cached = { version = "0.47.0", features = ["async", "serde", "ahash"] }

crates/middleware/src/thesaurus/mod.rs

@@ -23,6 +23,7 @@
 
 use crate::Result;
 use cached::proc_macro::cached;
+use persistence::Persistable;
 use std::collections::HashSet;
 use std::path::Path;
 use std::path::PathBuf;
@@ -58,9 +59,12 @@ pub async fn create_thesaurus_from_haystack(
         );
 
         let logseq = Logseq::default();
-        let thesaurus = logseq.build(&haystack.path).await?;
+        let thesaurus = logseq.build(role_name.clone(), &haystack.path).await?;
+        match thesaurus.save().await {
+            Ok(_) => log::debug!("Thesaurus saved"),
+            Err(e) => log::error!("Failed to save thesaurus: {:?}", e),
+        }
 
-        // Write thesaurus to local file (for now)
         let thesaurus_path = haystack.path.join("thesaurus.json");
 
         let thesaurus_json = serde_json::to_string_pretty(&thesaurus)?;
@@ -77,12 +81,17 @@ pub async fn create_thesaurus_from_haystack(
 /// resources (e.g. files) with key-value pairs and returns a `Thesaurus`
 /// (a dictionary with synonyms which map to higher-level concepts)
 pub trait ThesaurusBuilder {
-    /// `haystack` is the root directory for building the thesaurus
-    /// (e.g. a directory of Logseq files)
+    /// Build the thesaurus from the data source
+    ///
+    /// * `name` is the name of the thesaurus
+    /// * `haystack` is the root directory for building the thesaurus
+    ///   (e.g. a directory of Logseq files)
+    ///
     // This could be generalized (e.g. to take a `Read` trait object
     // or a `Resource` or a glob of inputs)
     fn build<P: Into<PathBuf> + Send>(
         &self,
+        name: String,
         haystack: P,
     ) -> impl std::future::Future<Output = Result<Thesaurus>> + Send;
 }
@@ -108,14 +117,14 @@ pub struct Logseq {
 impl ThesaurusBuilder for Logseq {
     /// Build the knowledge graph from the data source
     /// and store it in each rolegraph.
-    async fn build<P: Into<PathBuf> + Send>(&self, haystack: P) -> Result<Thesaurus> {
+    async fn build<P: Into<PathBuf> + Send>(&self, name: String, haystack: P) -> Result<Thesaurus> {
         let haystack = haystack.into();
         let messages = self
             .service
             .get_raw_messages(LOGSEQ_KEY_VALUE_DELIMITER, &haystack)
             .await?;
 
-        let thesaurus = index_inner(messages);
+        let thesaurus = index_inner(name, messages);
         Ok(thesaurus)
     }
 }
@@ -200,8 +209,8 @@ impl LogseqService {
 // This is a free-standing function because it's a requirement for caching the
 // results
 #[cached]
-fn index_inner(messages: Vec<Message>) -> Thesaurus {
-    let mut thesaurus = Thesaurus::new();
+fn index_inner(name: String, messages: Vec<Message>) -> Thesaurus {
+    let mut thesaurus = Thesaurus::new(name);
     let mut current_concept: Option<Concept> = None;
 
     let mut existing_paths: HashSet<PathBuf> = HashSet::new();

crates/middleware/tests/logseq.rs

@@ -11,7 +11,9 @@ mod tests {
     /// Uses `fixtures/logseq` as the haystack
     async fn test_logseq_thesaurus() -> Result<()> {
         let logseq = Logseq::default();
-        let thesaurus = logseq.build("fixtures/logseq").await?;
+        let thesaurus = logseq
+            .build("some_role".to_string(), "fixtures/logseq")
+            .await?;
         let json = serde_json::to_string_pretty(&thesaurus)?;
         println!("{}", json);
 

crates/middleware/tests/ripgrep.rs

@@ -22,7 +22,7 @@ mod tests {
         println!("Searching articles with query: {search_query:?} {role_name}");
 
         let cached_articles = search_haystacks(config_state.clone(), search_query.clone()).await?;
-        let docs: Vec<IndexedDocument> = config_state.search_articles(search_query).await;
+        let docs: Vec<IndexedDocument> = config_state.search_articles(&search_query).await;
         let articles = merge_and_serialize(cached_articles, docs);
         log::debug!("Final articles: {articles:?}");
 

crates/persistence/Cargo.toml

@@ -9,6 +9,7 @@ tracing-subscriber = { version = "0.3", features = [
   "env-filter",
   "tracing-log",
 ] }
+terraphim_types = { path = "../../terraphim_types" }
 
 async-once-cell = "0.5.3"
 async-trait = "0.1.74"
@@ -43,4 +44,3 @@ services-rocksdb = ["opendal/services-rocksdb"]
 services-sled = ["opendal/services-sled"]
 # Enable services atomic server
 services-atomicserver = ["opendal/services-atomicserver"]
-

crates/persistence/examples/simple_struct.rs

@@ -10,11 +10,8 @@ struct MyStruct {
 }
 #[async_trait]
 impl Persistable for MyStruct {
-    fn new() -> Self {
-        MyStruct {
-            name: String::new(),
-            age: 0,
-        }
+    fn new(name: String) -> Self {
+        MyStruct { name, age: 0 }
     }
 
     async fn save_to_one(&self, profile_name: &str) -> Result<()> {
@@ -55,7 +52,7 @@ async fn main() -> Result<()> {
     let (_ops, fastest_op) = obj.load_config().await?;
     println!("fastest_op: {:#?}", fastest_op);
 
-    let mut obj1 = MyStruct::new();
+    let mut obj1 = MyStruct::new("obj".to_string());
     let key = obj.get_key();
     println!("key: {}", key);
     obj1 = obj1.load(&key).await?;

crates/persistence/src/lib.rs

@@ -1,5 +1,6 @@
 pub mod error;
 pub mod settings;
+pub mod thesaurus;
 
 use async_once_cell::OnceCell as AsyncOnceCell;
 use async_trait::async_trait;
@@ -54,7 +55,7 @@ async fn init_device_storage() -> Result<DeviceStorage> {
 #[async_trait]
 pub trait Persistable: Serialize + DeserializeOwned {
     /// Create a new instance
-    fn new() -> Self;
+    fn new(key: String) -> Self;
 
     /// Save to all profiles
     async fn save(&self) -> Result<()>;
@@ -79,6 +80,7 @@ pub trait Persistable: Serialize + DeserializeOwned {
         let key = self.get_key();
         let serde_str = serde_json::to_string(&self)?;
         for (op, _time) in ops.values() {
+            log::debug!("Saving to operator: {:?}", op);
             op.write(&key, serde_str.clone()).await?;
         }
         Ok(())

crates/persistence/src/thesaurus.rs

@@ -0,0 +1,37 @@
+use crate::Result;
+use async_trait::async_trait;
+use terraphim_types::Thesaurus;
+
+use crate::Persistable;
+
+#[async_trait]
+impl Persistable for Thesaurus {
+    fn new(key: String) -> Self {
+        Thesaurus::new(key)
+    }
+
+    /// Save to a single profile
+    async fn save_to_one(&self, profile_name: &str) -> Result<()> {
+        self.save_to_profile(profile_name).await?;
+        Ok(())
+    }
+
+    // Saves to all profiles
+    async fn save(&self) -> Result<()> {
+        let _op = &self.load_config().await?.1;
+        let _ = self.save_to_all().await?;
+        Ok(())
+    }
+
+    /// Load key from the fastest operator
+    async fn load(&mut self, key: &str) -> Result<Self> {
+        let op = &self.load_config().await?.1;
+        let obj = self.load_from_operator(key, op).await?;
+        Ok(obj)
+    }
+
+    /// returns ulid as key + .json
+    fn get_key(&self) -> String {
+        format!("thesaurus_{}.json", self.name)
+    }
+}

crates/settings/src/lib.rs

@@ -17,6 +17,12 @@ pub enum Error {
 // which gets used by the `#[config]` macro below.
 pub type SettingsResult<T> = std::result::Result<T, Error>;
 
+/// Default config path
+pub const DEFAULT_CONFIG_PATH: &str = ".config";
+
+/// Default settings file
+pub const DEFAULT_SETTINGS: &str = include_str!("../default/settings_local.toml");
+
 /// Configuration settings for the device or server.
 ///
 /// These values are set when the server initializes, and do not change while
@@ -34,23 +40,28 @@ pub struct Settings {
 }
 
 impl Settings {
+    /// Get the default path for the config file
+    ///
+    /// This is the default path where the config file is stored.
+    pub fn default_config_path() -> PathBuf {
+        if let Some(proj_dirs) = ProjectDirs::from("com", "aks", "terraphim") {
+            let config_dir = proj_dirs.config_dir();
+            config_dir.to_path_buf()
+        } else {
+            PathBuf::from(DEFAULT_CONFIG_PATH)
+        }
+    }
+
     /// Load settings from environment and file
     pub fn load_from_env_and_file(config_path: Option<PathBuf>) -> SettingsResult<Self> {
         log::info!("Loading device settings...");
         let config_path = match config_path {
             Some(path) => path,
-            None => {
-                if let Some(proj_dirs) = ProjectDirs::from("com", "aks", "terraphim") {
-                    let config_dir = proj_dirs.config_dir();
-                    config_dir.to_path_buf()
-                } else {
-                    PathBuf::from(".config/")
-                }
-            }
+            None => Settings::default_config_path(),
         };
         log::debug!("Using config path: {:?}", config_path);
         let config_file = init_config_file(&config_path)?;
-        log::info!("Using config_file: {:?}", config_file);
+        log::debug!("Using config_file: {:?}", config_file);
 
         Ok(Settings::with_layers(&[
             Layer::Toml(config_file),
@@ -65,12 +76,10 @@ fn init_config_file(path: &PathBuf) -> Result<PathBuf, std::io::Error> {
     if !path.exists() {
         std::fs::create_dir_all(path)?;
     }
-    log::info!("Checking for settings.toml");
     let config_file = path.join("settings.toml");
     if !config_file.exists() {
-        log::warn!("Creating default config at: {:?}", config_file);
-        let default_config = include_str!("../default/settings_local.toml");
-        std::fs::write(&config_file, default_config)?;
+        log::info!("Creating default config at: {:?}", config_file);
+        std::fs::write(&config_file, DEFAULT_SETTINGS)?;
     }
     Ok(config_file)
 }

crates/terraphim_rolegraph/src/lib.rs

@@ -187,8 +187,8 @@ impl RoleGraph {
         }
 
         let mut ranked_documents = results.into_iter().collect::<Vec<_>>();
-        ranked_documents.sort_by_key(|&(_, ref doc)| std::cmp::Reverse(doc.rank));
-        ranked_documents.sort_by_key(|&(_, ref doc)| std::cmp::Reverse(doc.id.clone()));
+        ranked_documents.sort_by_key(|(_, doc)| std::cmp::Reverse(doc.rank));
+        ranked_documents.sort_by_key(|(_, doc)| std::cmp::Reverse(doc.id.clone()));
 
         let documents: Vec<_> = ranked_documents
             .into_iter()

terraphim_types/src/lib.rs

@@ -326,38 +326,46 @@ impl Node {
 /// where a resource can be as diverse as a Markdown file or a document in
 /// Notion or AtomicServer
 #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default)]
-pub struct Thesaurus(AHashMap<NormalizedTermValue, NormalizedTerm>);
+pub struct Thesaurus {
+    /// Name of the thesaurus
+    pub name: String,
+    /// The inner hashmap of normalized terms
+    inner: AHashMap<NormalizedTermValue, NormalizedTerm>,
+}
 
 impl Thesaurus {
     /// Create a new, empty thesaurus
-    pub fn new() -> Self {
-        Self(AHashMap::new())
+    pub fn new(name: String) -> Self {
+        Self {
+            name,
+            inner: AHashMap::new(),
+        }
     }
 
     /// Inserts a key-value pair into the thesaurus.
     pub fn insert(&mut self, key: NormalizedTermValue, value: NormalizedTerm) {
-        self.0.insert(key, value);
+        self.inner.insert(key, value);
     }
 
     /// Get the length of the thesaurus
     pub fn len(&self) -> usize {
-        self.0.len()
+        self.inner.len()
     }
 
     /// Check if the thesaurus is empty
     pub fn is_empty(&self) -> bool {
-        self.0.is_empty()
+        self.inner.is_empty()
     }
 
     /// Custom `get` method for the thesaurus, which accepts a
     /// `NormalizedTermValue` and returns a reference to the
     /// `NormalizedTerm`.
     pub fn get(&self, key: &NormalizedTermValue) -> Option<&NormalizedTerm> {
-        self.0.get(key)
+        self.inner.get(key)
     }
 
     pub fn keys(&self) -> std::collections::hash_map::Keys<NormalizedTermValue, NormalizedTerm> {
-        self.0.keys()
+        self.inner.keys()
     }
 }
 
@@ -367,7 +375,7 @@ impl<'a> IntoIterator for &'a Thesaurus {
     type IntoIter = Iter<'a, NormalizedTermValue, NormalizedTerm>;
 
     fn into_iter(self) -> Self::IntoIter {
-        self.0.iter()
+        self.inner.iter()
     }
 }