Add changes msg

src/config.rs

@@ -75,6 +75,7 @@ impl ThrottleConfig
 /// - ```static_content: Option<bool>```: all content is immutably cached at launch
 /// - ```ignore_regexes: Option<Vec<String>>```: do not serve content matching any of these patterns
 /// - ```generate_sitemap: Option<bool>```: sitemap.xml will be automatically generated (and updated) 
+/// - ```message_on_sitemap_reload: Option<bool>```: optionally send Discord notifications when sitemap is reloaded
 #[derive(Clone, Serialize, Deserialize)]
 pub struct ContentConfig
 {
@@ -85,7 +86,8 @@ pub struct ContentConfig
     pub browser_cache_period_seconds: u16,
     pub server_cache_period_seconds: u16,
     pub static_content: Option<bool>,
-    pub generate_sitemap: Option<bool>
+    pub generate_sitemap: Option<bool>,
+    pub message_on_sitemap_reload: Option<bool>
 }
 
 impl ContentConfig
@@ -101,7 +103,8 @@ impl ContentConfig
             browser_cache_period_seconds: 3600,
             server_cache_period_seconds: 3600,
             static_content: Some(false),
-            generate_sitemap: Some(true)
+            generate_sitemap: Some(true),
+            message_on_sitemap_reload: Some(false)
         }
     }
 }

src/content/sitemap.rs

@@ -92,6 +92,19 @@ impl ContentTree
         sha.finish().to_vec()
     }
 
+    pub fn collect_uris(&self) -> Vec<String>
+    {
+        let mut content: Vec<Content> = self.contents.clone().into_iter().map(|(x, _)| x).collect();
+        content.sort_by(|a, b| a.get_uri().cmp(&b.get_uri()));
+
+        let mut uris: Vec<String> = content.into_iter().map(|c| c.get_uri()).collect();
+        for (_, child) in &self.children
+        {
+            uris.append(&mut child.collect_uris());
+        }
+        uris
+    }
+
     pub fn push(&mut self, uri_stem: String, content: Content)
     {
         if uri_stem == "/"
@@ -290,6 +303,11 @@ impl SiteMap
         self.hash = self.contents.calculate_hash(false);
     }
 
+    pub fn collect_uris(&self) -> Vec<String>
+    {
+        self.contents.collect_uris()
+    }
+
     /// Searches the content path from [SiteMap::new] for [Content]
     ///  robots.txt and sitemap.xml can be generated and added here
     pub fn build

src/integrations/discord/post.rs

@@ -15,11 +15,11 @@ use crate::integrations::webhook::Webhook;
 /// # Example
 /// ```rust
 /// 
-/// use busser::integrations::{webhook::Webhook, discord::post::post};
+/// use busser::integrations::{webhook::Webhook, discord::post::post_message};
 /// 
 /// pub async fn post_to_discord(){
 ///     let w = Webhook::new("https://discord.com/api/webhooks/xxx/yyy".to_string());
-///     post(&w, "this is some plaintext".to_string());
+///     post_message(&w, &"this is some plaintext".to_string());
 /// }
 /// ```
 /// 
@@ -34,9 +34,9 @@ use crate::integrations::webhook::Webhook;
 ///  {"content": "this is some plaintext"}
 /// ``` 
 
-pub async fn post(w: &Webhook, msg: String) -> Result<String, reqwest::Error>
+/// Post a text message to a discord Webhook
+pub async fn post_message(w: &Webhook, msg: &String) -> Result<String, reqwest::Error>
 {
-
     crate::debug(format!("Posting to Discord {:?}", msg), None);
     let client = reqwest::Client::new();
 
@@ -51,5 +51,18 @@ pub async fn post(w: &Webhook, msg: String) -> Result<String, reqwest::Error>
         Ok(r) => Ok(format!("OK\nGot response:\n\n{:#?}", r)),
         Err(e) => Err(e)
     }
+}
 
+/// Attempt to post a message to the discord webhook
+pub async fn try_post(webhook: Option<Webhook>, msg: &String)
+{
+    match webhook
+    {
+        Some(w) => match post_message(&w, msg).await
+            {
+                Ok(_s) => (),
+                Err(e) => {crate::debug(format!("Error posting to discord\n{}", e), None);}
+            },
+        None => {crate::debug(format!("Discord webhook is None"), None);}
+    }
 }

src/main.rs

@@ -2,8 +2,10 @@ use std::time::Duration;
 
 use busser::config::{Config, CONFIG_PATH};
 use busser::content::sitemap::SiteMap;
+use busser::integrations::discord::post::try_post;
 use busser::server::http::ServerHttp;
 use busser::server::https::Server;
+use busser::util::formatted_differences;
 use busser::{openssl_version, program_version};
 use tokio::task::spawn;
 
@@ -56,32 +58,42 @@ async fn main() {
 ///  every [busser::config::ContentConfig::server_cache_period_seconds] the sitemap
 ///  hash (see [busser::content::sitemap::SiteMap::get_hash]) is checked, if it is 
 ///  different the server is re-served. 
+/// 
+///  On a re-serve if [busser::config::ContentConfig::message_on_sitemap_reload] is true
+///   A status message with (uri) additions and removals will be posted to Discord.
 async fn serve_observed(insert_tag: bool)
 {
-    let sitemap = SiteMap::from_config(&Config::load_or_default(CONFIG_PATH), insert_tag, false);
+    let mut sitemap = SiteMap::from_config(&Config::load_or_default(CONFIG_PATH), insert_tag, false);
     let mut hash = sitemap.get_hash();
 
-    let server = Server::new(0,0,0,0,sitemap);
+    let server = Server::new(0,0,0,0,sitemap.clone());
     let mut server_handle = server.get_handle();
     let mut thread_handle = spawn(async move {server.serve()}.await);
 
     loop
     {
         let config = Config::load_or_default(CONFIG_PATH);
-        let sitemap = SiteMap::from_config(&config, insert_tag, false);
-        let sitemap_hash = sitemap.get_hash();
+        let new_sitemap = SiteMap::from_config(&config, insert_tag, false);
+        let sitemap_hash = new_sitemap.get_hash();
 
         if sitemap_hash != hash
         {
             busser::debug(format!("Sitemap changed, shutting down"), None);
             server_handle.shutdown();
             thread_handle.abort();
 
-            let server = Server::new(0,0,0,0,sitemap);
+            let diffs = formatted_differences(new_sitemap.collect_uris(), sitemap.collect_uris());
+            sitemap = new_sitemap.clone();
+
+            let server = Server::new(0,0,0,0,new_sitemap);
             server_handle = server.get_handle();
             thread_handle = spawn(async move {server.serve()}.await);
             hash = sitemap_hash;
-            busser::debug(format!("Re-served"), None);
+            busser::debug(format!("Re-served\n Diffs:\n{}", diffs), None);
+            if config.content.message_on_sitemap_reload.is_some_and(|x|x)
+            {
+                try_post(config.notification_endpoint, &format!("The sitemap was refreshed with diffs:\n```{}```", diffs)).await;
+            }
         }
         busser::debug(format!("Next sitemap check: {}s", config.content.server_cache_period_seconds), None);
         tokio::time::sleep(Duration::from_secs(config.content.server_cache_period_seconds.into())).await;

src/server/api/stats.rs

@@ -6,7 +6,7 @@ use reqwest::StatusCode;
 use serde::Deserialize;
 use tokio::sync::Mutex;
 
-use crate::{config::{read_config, CONFIG_PATH}, server::stats::{digest::{digest_message, process_hits}, hits::HitStats}, integrations::{discord::post::post, is_authentic}};
+use crate::{config::{read_config, CONFIG_PATH}, integrations::{discord::post::try_post, is_authentic}, server::stats::{digest::{digest_message, process_hits}, hits::HitStats}};
 
 use super::ApiRequest;
 
@@ -145,20 +145,15 @@ impl ApiRequest for StatsDigest
             None => None
         };
 
-        let digest = process_hits(config.stats.path, from,to,config.stats.top_n_digest,stats);
-        let msg = digest_message(digest, from, to);
+        let msg = digest_message(process_hits(config.stats.path, from,to,config.stats.top_n_digest,stats), from, to);
 
         if self.payload.post_discord
         {
-            match config.notification_endpoint
-            {
-                Some(endpoint) => match post(&endpoint, msg.clone()).await
-                    {
-                        Ok(_s) => (),
-                        Err(e) => {crate::debug(format!("Error posting to discord\n{}", e), None);}
-                    },
-                None => ()
-            }
+            try_post
+            (
+                config.notification_endpoint,
+                &msg
+            ).await;
         }
 
         (Some(msg), StatusCode::OK)

src/server/stats/mod.rs

@@ -5,7 +5,7 @@ use chrono::{DateTime, Utc};
 use cron::Schedule;
 use tokio::sync::Mutex;
 
-use crate::{config::{read_config, Config, CONFIG_PATH}, filesystem::file::File, integrations::discord::post::post, task::{next_job_time, schedule_from_option, Task}};
+use crate::{config::{Config, CONFIG_PATH}, filesystem::file::File, integrations::discord::post::try_post, task::{next_job_time, schedule_from_option, Task}};
 
 use self::{digest::{digest_message, process_hits}, file::StatsFile, hits::HitStats};
 
@@ -132,14 +132,7 @@ impl Task for StatsDigestTask
         {
             let mut stats = self.state.lock().await;
 
-            let config = match read_config(CONFIG_PATH)
-            {
-                Some(c) => c,
-                None =>
-                {
-                    Config::default()
-                }
-            };
+            let config = Config::load_or_default(CONFIG_PATH);
 
             stats.summary = process_hits
             (
@@ -150,16 +143,11 @@ impl Task for StatsDigestTask
                 Some(stats.to_owned())
             );
 
-            let msg = digest_message(stats.summary.clone(), Some(self.last_run), None);
-            match config.notification_endpoint 
-            {
-                Some(endpoint) => match post(&endpoint, msg).await
-                    {
-                        Ok(_s) => (),
-                        Err(e) => {crate::debug(format!("Error posting to discord\n{}", e), None);}
-                    },
-                None => ()
-            }
+            try_post
+            (
+                config.notification_endpoint,
+                &digest_message(stats.summary.clone(), Some(self.last_run), None)
+            ).await;
         }
 
         let config = Config::load_or_default(CONFIG_PATH);

src/util.rs

@@ -1,5 +1,5 @@
 use core::fmt;
-use std::{fmt::Write, io::{Read, Write as ioWrite}, time::Instant};
+use std::{collections::HashSet, fmt::Write, io::{Read, Write as ioWrite}, time::Instant};
 use chrono::{DateTime, Datelike, FixedOffset};
 use libflate::deflate::{Encoder, Decoder};
 use openssl::sha::Sha256;
@@ -168,4 +168,37 @@ pub fn date_now() -> String
 pub fn date_to_rfc3339(date: &str) -> Result<DateTime<FixedOffset>, chrono::ParseError>
 {
     DateTime::parse_from_rfc3339(format!("{}T00:00:00+00:00", date).as_str())
+}
+
+pub fn differences(new: Vec<String>, old: Vec<String>) -> (Vec<String>, Vec<String>)
+{
+    let hnew: HashSet<String> = new.into_iter().collect();
+    let hold: HashSet<String> = old.into_iter().collect();
+
+    let new_values = hnew.difference(&hold);
+    let lost_values = hold.difference(&hnew);
+
+    let mut added: Vec<String> = new_values.cloned().collect();
+    let mut removed: Vec<String> = lost_values.cloned().collect();
+
+    added.sort();
+    removed.sort();
+
+    (added, removed)
+}
+
+pub fn formatted_differences(new: Vec<String>, old: Vec<String>) -> String
+{
+    let (added, removed) = differences(new, old);
+    let mut diffs = String::new();
+    for add in added
+    {
+        diffs = format!("{}+ {}\n", diffs, add);
+    }
+    for rm in removed
+    {
+        diffs = format!("{}- {}\n", diffs, rm);
+    }
+
+    diffs
 }

tests/test_config.rs

@@ -85,6 +85,7 @@ mod config
         assert_eq!(content.browser_cache_period_seconds, 3600);
         assert_eq!(content.server_cache_period_seconds, 3600);
         assert_eq!(content.static_content, Some(false));
+        assert_eq!(content.message_on_sitemap_reload, Some(false));
 
         let config = Config::default();
 

tests/test_discord.rs

@@ -3,7 +3,7 @@ mod common;
 #[cfg(test)]
 mod discord
 {
-    use busser::integrations::{discord::post::post, webhook::Webhook};
+    use busser::integrations::{discord::post::post_message, webhook::Webhook};
 
     #[tokio::test]
     async fn test_webhook()
@@ -12,13 +12,13 @@ mod discord
 
         assert_eq!(w.get_addr(), "https://discord.com/api/webhooks/xxx/yyy");
 
-        assert!(post(&w, "400".to_string()).await.is_ok());
+        assert!(post_message(&w, &"400".to_string()).await.is_ok());
     }
 
     #[tokio::test]
     async fn test_err_webhook()
     {
         let w = Webhook::new("not_a_domain".to_string());
-        assert!(post(&w, "400".to_string()).await.is_err());
+        assert!(post_message(&w, &"400".to_string()).await.is_err());
     }
 }

tests/test_sitemap.rs

@@ -30,12 +30,23 @@ mod sitemap
         let empty_sitemap = r#"<?xml version="1.0" encoding="UTF-8"?>
 <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:image="http://www.google.com/schemas/sitemap-image/1.1" xmlns:video="http://www.google.com/schemas/sitemap-video/1.1"></urlset>"#;
         let mut sitemap = SiteMap::new("https://test.domain".to_owned(), "tests/pages".to_owned());
+
         assert_eq!(empty_sitemap, String::from_utf8(sitemap.to_xml()).unwrap());
+        assert_eq!(sitemap.collect_uris(), Vec::<String>::new());
+
         sitemap.build(true, false, None);
 
         assert!(Path::new("tests/pages/robots.txt").exists());
         assert!(Path::new("tests/pages/sitemap.xml").exists());
 
+        let uris = sitemap.collect_uris();
+        assert!(uris.contains(&"/a".to_string()));
+        assert!(uris.contains(&"/b".to_string()));
+        assert!(uris.contains(&"/c/d".to_string()));
+        assert!(uris.contains(&"/a.html".to_string()));
+        assert!(uris.contains(&"/b.html".to_string()));
+        assert!(uris.contains(&"/c/d.html".to_string()));
+
         let sitemap_disk = read_file_utf8("tests/pages/sitemap.xml").unwrap();
         let robots_disk = read_file_utf8("tests/pages/robots.txt").unwrap();
 

tests/test_utils.rs

@@ -3,7 +3,7 @@ mod common;
 #[cfg(test)]
 mod util
 {
-    use busser::util::{date_now, date_to_rfc3339, hash, matches_one, read_bytes, strip_control_characters};
+    use busser::util::{date_now, date_to_rfc3339, differences, formatted_differences, hash, matches_one, read_bytes, strip_control_characters};
 
     use busser::util::{compress, compress_string, decompress, decompress_utf8_string};
     use chrono::{DateTime, Datelike};
@@ -86,4 +86,30 @@ mod util
             assert_eq!(strip_control_characters(test_string), "a_test_string");
         }
     }
+
+    #[test]
+    fn test_differences()
+    {
+        let old = vec!["a".to_string(), "b".to_string()];
+        let new = vec!["b".to_string(), "c".to_string()];
+
+        let (new, lost) = differences(new, old);
+
+        assert_eq!(new, vec!["c".to_string()]);
+        assert_eq!(lost, vec!["a".to_string()]);
+    }
+
+    #[test]
+    fn test_formatted_differences()
+    {
+        let old = vec!["a".to_string(), "b".to_string()];
+        let new = vec!["b".to_string(), "c".to_string(), "d".to_string()];
+
+        let diffs = formatted_differences(new, old);
+        let expected = r#"+ c
++ d
+- a
+"#;
+        assert_eq!(diffs, expected);
+    }
 }