diff --git a/src/config.rs b/src/config.rs index f93cf8a..4ff7e03 100644 --- a/src/config.rs +++ b/src/config.rs @@ -75,6 +75,7 @@ impl ThrottleConfig /// - ```static_content: Option```: all content is immutably cached at launch /// - ```ignore_regexes: Option>```: do not serve content matching any of these patterns /// - ```generate_sitemap: Option```: sitemap.xml will be automatically generated (and updated) +/// - ```message_on_sitemap_reload: Option```: optionally send Discord notifications when sitemap is reloaded #[derive(Clone, Serialize, Deserialize)] pub struct ContentConfig { @@ -85,7 +86,8 @@ pub struct ContentConfig pub browser_cache_period_seconds: u16, pub server_cache_period_seconds: u16, pub static_content: Option, - pub generate_sitemap: Option + pub generate_sitemap: Option, + pub message_on_sitemap_reload: Option } impl ContentConfig @@ -101,7 +103,8 @@ impl ContentConfig browser_cache_period_seconds: 3600, server_cache_period_seconds: 3600, static_content: Some(false), - generate_sitemap: Some(true) + generate_sitemap: Some(true), + message_on_sitemap_reload: Some(false) } } } diff --git a/src/content/sitemap.rs b/src/content/sitemap.rs index 04fea0a..3a2c687 100644 --- a/src/content/sitemap.rs +++ b/src/content/sitemap.rs @@ -92,6 +92,19 @@ impl ContentTree sha.finish().to_vec() } + pub fn collect_uris(&self) -> Vec + { + let mut content: Vec = self.contents.clone().into_iter().map(|(x, _)| x).collect(); + content.sort_by(|a, b| a.get_uri().cmp(&b.get_uri())); + + let mut uris: Vec = content.into_iter().map(|c| c.get_uri()).collect(); + for (_, child) in &self.children + { + uris.append(&mut child.collect_uris()); + } + uris + } + pub fn push(&mut self, uri_stem: String, content: Content) { if uri_stem == "/" @@ -290,6 +303,11 @@ impl SiteMap self.hash = self.contents.calculate_hash(false); } + pub fn collect_uris(&self) -> Vec + { + self.contents.collect_uris() + } + /// Searches the content path from [SiteMap::new] for [Content] /// robots.txt and sitemap.xml can be generated and added here pub fn build diff --git a/src/integrations/discord/post.rs b/src/integrations/discord/post.rs index 1b21eeb..1746a4f 100644 --- a/src/integrations/discord/post.rs +++ b/src/integrations/discord/post.rs @@ -15,11 +15,11 @@ use crate::integrations::webhook::Webhook; /// # Example /// ```rust /// -/// use busser::integrations::{webhook::Webhook, discord::post::post}; +/// use busser::integrations::{webhook::Webhook, discord::post::post_message}; /// /// pub async fn post_to_discord(){ /// let w = Webhook::new("https://discord.com/api/webhooks/xxx/yyy".to_string()); -/// post(&w, "this is some plaintext".to_string()); +/// post_message(&w, &"this is some plaintext".to_string()); /// } /// ``` /// @@ -34,9 +34,9 @@ use crate::integrations::webhook::Webhook; /// {"content": "this is some plaintext"} /// ``` -pub async fn post(w: &Webhook, msg: String) -> Result +/// Post a text message to a discord Webhook +pub async fn post_message(w: &Webhook, msg: &String) -> Result { - crate::debug(format!("Posting to Discord {:?}", msg), None); let client = reqwest::Client::new(); @@ -51,5 +51,18 @@ pub async fn post(w: &Webhook, msg: String) -> Result Ok(r) => Ok(format!("OK\nGot response:\n\n{:#?}", r)), Err(e) => Err(e) } +} +/// Attempt to post a message to the discord webhook +pub async fn try_post(webhook: Option, msg: &String) +{ + match webhook + { + Some(w) => match post_message(&w, msg).await + { + Ok(_s) => (), + Err(e) => {crate::debug(format!("Error posting to discord\n{}", e), None);} + }, + None => {crate::debug(format!("Discord webhook is None"), None);} + } } \ No newline at end of file diff --git a/src/main.rs b/src/main.rs index 6d867d4..41b9986 100644 --- a/src/main.rs +++ b/src/main.rs @@ -2,8 +2,10 @@ use std::time::Duration; use busser::config::{Config, CONFIG_PATH}; use busser::content::sitemap::SiteMap; +use busser::integrations::discord::post::try_post; use busser::server::http::ServerHttp; use busser::server::https::Server; +use busser::util::formatted_differences; use busser::{openssl_version, program_version}; use tokio::task::spawn; @@ -56,20 +58,23 @@ async fn main() { /// every [busser::config::ContentConfig::server_cache_period_seconds] the sitemap /// hash (see [busser::content::sitemap::SiteMap::get_hash]) is checked, if it is /// different the server is re-served. +/// +/// On a re-serve if [busser::config::ContentConfig::message_on_sitemap_reload] is true +/// A status message with (uri) additions and removals will be posted to Discord. async fn serve_observed(insert_tag: bool) { - let sitemap = SiteMap::from_config(&Config::load_or_default(CONFIG_PATH), insert_tag, false); + let mut sitemap = SiteMap::from_config(&Config::load_or_default(CONFIG_PATH), insert_tag, false); let mut hash = sitemap.get_hash(); - let server = Server::new(0,0,0,0,sitemap); + let server = Server::new(0,0,0,0,sitemap.clone()); let mut server_handle = server.get_handle(); let mut thread_handle = spawn(async move {server.serve()}.await); loop { let config = Config::load_or_default(CONFIG_PATH); - let sitemap = SiteMap::from_config(&config, insert_tag, false); - let sitemap_hash = sitemap.get_hash(); + let new_sitemap = SiteMap::from_config(&config, insert_tag, false); + let sitemap_hash = new_sitemap.get_hash(); if sitemap_hash != hash { @@ -77,11 +82,18 @@ async fn serve_observed(insert_tag: bool) server_handle.shutdown(); thread_handle.abort(); - let server = Server::new(0,0,0,0,sitemap); + let diffs = formatted_differences(new_sitemap.collect_uris(), sitemap.collect_uris()); + sitemap = new_sitemap.clone(); + + let server = Server::new(0,0,0,0,new_sitemap); server_handle = server.get_handle(); thread_handle = spawn(async move {server.serve()}.await); hash = sitemap_hash; - busser::debug(format!("Re-served"), None); + busser::debug(format!("Re-served\n Diffs:\n{}", diffs), None); + if config.content.message_on_sitemap_reload.is_some_and(|x|x) + { + try_post(config.notification_endpoint, &format!("The sitemap was refreshed with diffs:\n```{}```", diffs)).await; + } } busser::debug(format!("Next sitemap check: {}s", config.content.server_cache_period_seconds), None); tokio::time::sleep(Duration::from_secs(config.content.server_cache_period_seconds.into())).await; diff --git a/src/server/api/stats.rs b/src/server/api/stats.rs index abc048e..b8f31ed 100644 --- a/src/server/api/stats.rs +++ b/src/server/api/stats.rs @@ -6,7 +6,7 @@ use reqwest::StatusCode; use serde::Deserialize; use tokio::sync::Mutex; -use crate::{config::{read_config, CONFIG_PATH}, server::stats::{digest::{digest_message, process_hits}, hits::HitStats}, integrations::{discord::post::post, is_authentic}}; +use crate::{config::{read_config, CONFIG_PATH}, integrations::{discord::post::try_post, is_authentic}, server::stats::{digest::{digest_message, process_hits}, hits::HitStats}}; use super::ApiRequest; @@ -145,20 +145,15 @@ impl ApiRequest for StatsDigest None => None }; - let digest = process_hits(config.stats.path, from,to,config.stats.top_n_digest,stats); - let msg = digest_message(digest, from, to); + let msg = digest_message(process_hits(config.stats.path, from,to,config.stats.top_n_digest,stats), from, to); if self.payload.post_discord { - match config.notification_endpoint - { - Some(endpoint) => match post(&endpoint, msg.clone()).await - { - Ok(_s) => (), - Err(e) => {crate::debug(format!("Error posting to discord\n{}", e), None);} - }, - None => () - } + try_post + ( + config.notification_endpoint, + &msg + ).await; } (Some(msg), StatusCode::OK) diff --git a/src/server/stats/mod.rs b/src/server/stats/mod.rs index e789b7e..8728fdb 100644 --- a/src/server/stats/mod.rs +++ b/src/server/stats/mod.rs @@ -5,7 +5,7 @@ use chrono::{DateTime, Utc}; use cron::Schedule; use tokio::sync::Mutex; -use crate::{config::{read_config, Config, CONFIG_PATH}, filesystem::file::File, integrations::discord::post::post, task::{next_job_time, schedule_from_option, Task}}; +use crate::{config::{Config, CONFIG_PATH}, filesystem::file::File, integrations::discord::post::try_post, task::{next_job_time, schedule_from_option, Task}}; use self::{digest::{digest_message, process_hits}, file::StatsFile, hits::HitStats}; @@ -132,14 +132,7 @@ impl Task for StatsDigestTask { let mut stats = self.state.lock().await; - let config = match read_config(CONFIG_PATH) - { - Some(c) => c, - None => - { - Config::default() - } - }; + let config = Config::load_or_default(CONFIG_PATH); stats.summary = process_hits ( @@ -150,16 +143,11 @@ impl Task for StatsDigestTask Some(stats.to_owned()) ); - let msg = digest_message(stats.summary.clone(), Some(self.last_run), None); - match config.notification_endpoint - { - Some(endpoint) => match post(&endpoint, msg).await - { - Ok(_s) => (), - Err(e) => {crate::debug(format!("Error posting to discord\n{}", e), None);} - }, - None => () - } + try_post + ( + config.notification_endpoint, + &digest_message(stats.summary.clone(), Some(self.last_run), None) + ).await; } let config = Config::load_or_default(CONFIG_PATH); diff --git a/src/util.rs b/src/util.rs index 31a23c2..a5f9b4d 100644 --- a/src/util.rs +++ b/src/util.rs @@ -1,5 +1,5 @@ use core::fmt; -use std::{fmt::Write, io::{Read, Write as ioWrite}, time::Instant}; +use std::{collections::HashSet, fmt::Write, io::{Read, Write as ioWrite}, time::Instant}; use chrono::{DateTime, Datelike, FixedOffset}; use libflate::deflate::{Encoder, Decoder}; use openssl::sha::Sha256; @@ -168,4 +168,37 @@ pub fn date_now() -> String pub fn date_to_rfc3339(date: &str) -> Result, chrono::ParseError> { DateTime::parse_from_rfc3339(format!("{}T00:00:00+00:00", date).as_str()) +} + +pub fn differences(new: Vec, old: Vec) -> (Vec, Vec) +{ + let hnew: HashSet = new.into_iter().collect(); + let hold: HashSet = old.into_iter().collect(); + + let new_values = hnew.difference(&hold); + let lost_values = hold.difference(&hnew); + + let mut added: Vec = new_values.cloned().collect(); + let mut removed: Vec = lost_values.cloned().collect(); + + added.sort(); + removed.sort(); + + (added, removed) +} + +pub fn formatted_differences(new: Vec, old: Vec) -> String +{ + let (added, removed) = differences(new, old); + let mut diffs = String::new(); + for add in added + { + diffs = format!("{}+ {}\n", diffs, add); + } + for rm in removed + { + diffs = format!("{}- {}\n", diffs, rm); + } + + diffs } \ No newline at end of file diff --git a/tests/test_config.rs b/tests/test_config.rs index 8134937..f0e61fa 100644 --- a/tests/test_config.rs +++ b/tests/test_config.rs @@ -85,6 +85,7 @@ mod config assert_eq!(content.browser_cache_period_seconds, 3600); assert_eq!(content.server_cache_period_seconds, 3600); assert_eq!(content.static_content, Some(false)); + assert_eq!(content.message_on_sitemap_reload, Some(false)); let config = Config::default(); diff --git a/tests/test_discord.rs b/tests/test_discord.rs index c6184d5..1c1018d 100644 --- a/tests/test_discord.rs +++ b/tests/test_discord.rs @@ -3,7 +3,7 @@ mod common; #[cfg(test)] mod discord { - use busser::integrations::{discord::post::post, webhook::Webhook}; + use busser::integrations::{discord::post::post_message, webhook::Webhook}; #[tokio::test] async fn test_webhook() @@ -12,13 +12,13 @@ mod discord assert_eq!(w.get_addr(), "https://discord.com/api/webhooks/xxx/yyy"); - assert!(post(&w, "400".to_string()).await.is_ok()); + assert!(post_message(&w, &"400".to_string()).await.is_ok()); } #[tokio::test] async fn test_err_webhook() { let w = Webhook::new("not_a_domain".to_string()); - assert!(post(&w, "400".to_string()).await.is_err()); + assert!(post_message(&w, &"400".to_string()).await.is_err()); } } \ No newline at end of file diff --git a/tests/test_sitemap.rs b/tests/test_sitemap.rs index 0658b15..9ad5060 100644 --- a/tests/test_sitemap.rs +++ b/tests/test_sitemap.rs @@ -30,12 +30,23 @@ mod sitemap let empty_sitemap = r#" "#; let mut sitemap = SiteMap::new("https://test.domain".to_owned(), "tests/pages".to_owned()); + assert_eq!(empty_sitemap, String::from_utf8(sitemap.to_xml()).unwrap()); + assert_eq!(sitemap.collect_uris(), Vec::::new()); + sitemap.build(true, false, None); assert!(Path::new("tests/pages/robots.txt").exists()); assert!(Path::new("tests/pages/sitemap.xml").exists()); + let uris = sitemap.collect_uris(); + assert!(uris.contains(&"/a".to_string())); + assert!(uris.contains(&"/b".to_string())); + assert!(uris.contains(&"/c/d".to_string())); + assert!(uris.contains(&"/a.html".to_string())); + assert!(uris.contains(&"/b.html".to_string())); + assert!(uris.contains(&"/c/d.html".to_string())); + let sitemap_disk = read_file_utf8("tests/pages/sitemap.xml").unwrap(); let robots_disk = read_file_utf8("tests/pages/robots.txt").unwrap(); diff --git a/tests/test_utils.rs b/tests/test_utils.rs index d1e8c18..7b8a65d 100644 --- a/tests/test_utils.rs +++ b/tests/test_utils.rs @@ -3,7 +3,7 @@ mod common; #[cfg(test)] mod util { - use busser::util::{date_now, date_to_rfc3339, hash, matches_one, read_bytes, strip_control_characters}; + use busser::util::{date_now, date_to_rfc3339, differences, formatted_differences, hash, matches_one, read_bytes, strip_control_characters}; use busser::util::{compress, compress_string, decompress, decompress_utf8_string}; use chrono::{DateTime, Datelike}; @@ -86,4 +86,30 @@ mod util assert_eq!(strip_control_characters(test_string), "a_test_string"); } } + + #[test] + fn test_differences() + { + let old = vec!["a".to_string(), "b".to_string()]; + let new = vec!["b".to_string(), "c".to_string()]; + + let (new, lost) = differences(new, old); + + assert_eq!(new, vec!["c".to_string()]); + assert_eq!(lost, vec!["a".to_string()]); + } + + #[test] + fn test_formatted_differences() + { + let old = vec!["a".to_string(), "b".to_string()]; + let new = vec!["b".to_string(), "c".to_string(), "d".to_string()]; + + let diffs = formatted_differences(new, old); + let expected = r#"+ c ++ d +- a +"#; + assert_eq!(diffs, expected); + } } \ No newline at end of file