From b4f40a3ac5a826ed6424cd3c489d3a781b022b13 Mon Sep 17 00:00:00 2001 From: 0xb10c Date: Tue, 20 Feb 2024 11:36:20 +0100 Subject: [PATCH 1/5] add: node reachabillity bool --- src/main.rs | 1 + src/types.rs | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/src/main.rs b/src/main.rs index 2d122b9..a956c2e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -304,6 +304,7 @@ async fn main() -> Result<(), MainError> { &relevant_tips, version_info.clone(), last_change_timestamp, + true, // here, the node was reachable ); let forks = headertree::recent_forks(&tree_clone, MAX_FORKS_IN_CACHE).await; diff --git a/src/types.rs b/src/types.rs index 6680084..1058007 100644 --- a/src/types.rs +++ b/src/types.rs @@ -144,6 +144,8 @@ pub struct NodeDataJson { pub last_changed_timestamp: u64, /// The node subversion as advertised by the node on the network. pub version: String, + /// If the last getchaintips RPC reached the node. + pub reachable: bool, } impl NodeDataJson { @@ -152,6 +154,7 @@ impl NodeDataJson { tips: &Vec, version: String, last_changed_timestamp: u64, + reachable: bool, ) -> Self { NodeDataJson { id: info.id, @@ -160,6 +163,7 @@ impl NodeDataJson { tips: tips.iter().map(TipInfoJson::new).collect(), last_changed_timestamp, version, + reachable, } } } From 6e3c6b8405b662876b0eeeb174329d36c5c4a4ec Mon Sep 17 00:00:00 2001 From: 0xb10c Date: Tue, 20 Feb 2024 12:24:05 +0100 Subject: [PATCH 2/5] add: populate cache with nodes from config previously, the nodes would only be added to the cache after a succesfull RPC request. We want to show the nodes even if the RPC requests fail. --- src/main.rs | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/src/main.rs b/src/main.rs index a956c2e..2f58bdb 100644 --- a/src/main.rs +++ b/src/main.rs @@ -110,16 +110,34 @@ async fn main() -> Result<(), MainError> { }, )); + // pupulate cache with available data let forks = headertree::recent_forks(&tree, MAX_FORKS_IN_CACHE).await; let header_infos_json = headertree::strip_tree(&tree, network.max_interesting_heights, BTreeSet::new()).await; { let mut locked_caches = caches.lock().await; + let node_data: NodeData = network + .nodes + .iter() + .cloned() + .map(|n| { + ( + n.info().id, + NodeDataJson::new( + n.info(), + &vec![], // no chain tips knows yet + VERSION_UNKNOWN.to_string(), // is updated later, when we know it + 0, // timestamp of last block update + true, // assume the node is reachable, if it isn't we set it to false after the first getchaintips RPC call anyway + ), + ) + }) + .collect(); locked_caches.insert( network.id, Cache { header_infos_json, - node_data: BTreeMap::new(), + node_data, forks, }, ); @@ -309,6 +327,7 @@ async fn main() -> Result<(), MainError> { let forks = headertree::recent_forks(&tree_clone, MAX_FORKS_IN_CACHE).await; { + // update node cache with new state data let mut locked_cache = caches_clone.lock().await; let this_network = locked_cache .get(&network.id) From 72fb4cc6149b0738061f951b0aeaaeebff176ec0 Mon Sep 17 00:00:00 2001 From: 0xb10c Date: Tue, 20 Feb 2024 13:05:05 +0100 Subject: [PATCH 3/5] fix: don't assume all nodes have an active tip --- www/js/blocktree.js | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/www/js/blocktree.js b/www/js/blocktree.js index e0236ab..fd7a0ab 100644 --- a/www/js/blocktree.js +++ b/www/js/blocktree.js @@ -381,11 +381,26 @@ function node_description_summary(description) { return description } +function get_active_height_or_0(node) { + let active_tips = node.tips.filter(tip => tip.status == "active") + if (active_tips.length > 0) { + return active_tips[0].height + } + return 0 +} + +function get_active_hash_or_fake(node) { + let active_tips = node.tips.filter(tip => tip.status == "active") + if (active_tips.length > 0) { + return active_tips[0].hash + } + return "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffdead" +} async function draw_nodes() { nodeInfoRow.html(null); nodeInfoRow.selectAll('.node-info') - .data(state_data.nodes.sort((a, b) => a.tips.filter(tip => tip.status == "active")[0].height - b.tips.filter(tip => tip.status == "active")[0].height)) + .data(state_data.nodes.sort((a, b) => get_active_height_or_0(a) - get_active_height_or_0(b))) .enter() .append("div") .attr("class", "row-cols-auto px-1") @@ -406,20 +421,20 @@ async function draw_nodes() { version: ${d.version}
- changed: ${new Date(d.last_changed_timestamp * 1000).toLocaleTimeString()} + changed: ${new Date(d.last_changed_timestamp * 1000).toLocaleString()}
-
- height: ${d.tips.filter(tip => tip.status == "active")[0].height} +
+ height: ${get_active_height_or_0(d)}
-
+
- tip: …${d.tips.filter(tip => tip.status == "active")[0].hash.substring(54, 64)} + tip: …${get_active_hash_or_fake(d).substring(54, 64)} - ${d.tips.filter(tip => tip.status == "active")[0].hash} + ${get_active_hash_or_fake(d)}
From 74da0cd6eb5e7c468b3cb6d68a4c7c79aa0565e5 Mon Sep 17 00:00:00 2001 From: 0xb10c Date: Tue, 20 Feb 2024 21:29:03 +0100 Subject: [PATCH 4/5] add: set node reachable & show on frontend --- src/main.rs | 68 ++++++++++++++++++++++++++++++++++++++++++++- src/types.rs | 4 +++ www/js/blocktree.js | 61 +++++++++++++++++++++++++++++----------- 3 files changed, 116 insertions(+), 17 deletions(-) diff --git a/src/main.rs b/src/main.rs index 2f58bdb..727ff45 100644 --- a/src/main.rs +++ b/src/main.rs @@ -196,7 +196,10 @@ async fn main() -> Result<(), MainError> { continue; }; let tips = match node.tips().await { - Ok(tips) => tips, + Ok(tips) => { + node_reachable(&caches_clone, network.id, node.info().id, true).await; + tips + } Err(e) => { error!( "Could not fetch chaintips from {} on network '{}' (id={}): {:?}", @@ -205,6 +208,7 @@ async fn main() -> Result<(), MainError> { network.id, e ); + node_reachable(&caches_clone, network.id, node.info().id, false).await; continue; } }; @@ -594,6 +598,16 @@ async fn main() -> Result<(), MainError> { Ok(()) } +async fn node_reachable(caches: &Caches, network_id: u32, node_id: u32, reachable: bool) { + let mut locked_cache = caches.lock().await; + locked_cache.entry(network_id).and_modify(|network| { + network + .node_data + .entry(node_id) + .and_modify(|e| e.reachable(reachable)); + }); +} + // Find out for which heights we have tips for. These are // interesting to us - we don't want strip them from the tree. // This includes tips that aren't from a fork, but rather from @@ -615,3 +629,55 @@ async fn tip_heights(network_id: u32, caches: &Caches) -> BTreeSet { } tip_heights } + +#[cfg(test)] +mod tests { + use super::*; + use crate::node::NodeInfo; + + async fn get_test_node_reachable(caches: &Caches, net_id: u32, node_id: u32) -> bool { + let locked_caches = caches.lock().await; + locked_caches + .get(&net_id) + .expect("network id should be there") + .node_data + .get(&node_id) + .expect("node id should be there") + .reachable + } + + #[tokio::test] + async fn test_node_reachable() { + let network_id: u32 = 0; + let caches: Caches = Arc::new(Mutex::new(BTreeMap::new())); + let node = NodeInfo { + id: 0, + name: "".to_string(), + description: "".to_string(), + }; + { + // populate data + let mut locked_caches = caches.lock().await; + let mut node_data: NodeData = BTreeMap::new(); + node_data.insert( + node.id, + NodeDataJson::new(node.clone(), &vec![], "".to_string(), 0, true), + ); + locked_caches.insert( + network_id, + Cache { + header_infos_json: vec![], + node_data, + forks: vec![], + }, + ); + } + assert_eq!(get_test_node_reachable(&caches, network_id, node.id).await, true); + + node_reachable(&caches, network_id, node.id, false).await; + assert_eq!(get_test_node_reachable(&caches, network_id, node.id).await, false); + + node_reachable(&caches, network_id, node.id, true).await; + assert_eq!(get_test_node_reachable(&caches, network_id, node.id).await, true); + } +} diff --git a/src/types.rs b/src/types.rs index 1058007..ccbfbe1 100644 --- a/src/types.rs +++ b/src/types.rs @@ -166,6 +166,10 @@ impl NodeDataJson { reachable, } } + + pub fn reachable(&mut self, r: bool) { + self.reachable = r; + } } #[derive(Serialize, Clone)] diff --git a/www/js/blocktree.js b/www/js/blocktree.js index fd7a0ab..e250dd6 100644 --- a/www/js/blocktree.js +++ b/www/js/blocktree.js @@ -397,6 +397,37 @@ function get_active_hash_or_fake(node) { return "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffdead" } +function ago(timestamp) { + const rtf = new Intl.RelativeTimeFormat("en", { + style: "narrow", + numeric: "always", + }); + const now = new Date() + const utc_seconds = (now.getTime() + now.getTimezoneOffset()*60) / 1000; + const seconds = timestamp - utc_seconds; + if (seconds > -90) { + return rtf.format(seconds, "seconds"); + } + const minutes = parseInt(seconds/60); + if (minutes > -60) { + return rtf.format(minutes, "minutes"); + } + const hours = parseInt(minutes/60); + if (hours > -24) { + return rtf.format(hours, "hours"); + } + const days = parseInt(hours/60); + if (days > -30) { + return rtf.format(days, "days"); + } + const months = parseInt(days/31); + if (months > -12) { + return rtf.format(months, "months"); + } + + return "a long time ago" +} + async function draw_nodes() { nodeInfoRow.html(null); nodeInfoRow.selectAll('.node-info') @@ -405,32 +436,30 @@ async function draw_nodes() { .append("div") .attr("class", "row-cols-auto px-1") .html(d => ` -
-
-
- - Node symbol - ${d.name} - -
- - ${node_description_summary(d.description)} +
+
+ + Node symbol + ${d.name} +
+
+ ${d.reachable ? "": "RPC unreachable"} + ${d.version.replaceAll("/", "").replaceAll("Satoshi:", "")} + tip changed ${ago(d.last_changed_timestamp)}
+
- version: ${d.version} -
-
- changed: ${new Date(d.last_changed_timestamp * 1000).toLocaleString()} + ${node_description_summary(d.description)}
height: ${get_active_height_or_0(d)}
-
+
- tip: …${get_active_hash_or_fake(d).substring(54, 64)} + tip hash: …${get_active_hash_or_fake(d).substring(54, 64)} From d4a9a50b3061d102d586add9256e1237fdfb2d45 Mon Sep 17 00:00:00 2001 From: 0xb10c Date: Tue, 20 Feb 2024 22:22:16 +0100 Subject: [PATCH 5/5] add: RSS feed for unreachable nodes --- src/main.rs | 41 +++++++++++++++++++++++---------- src/rss.rs | 62 ++++++++++++++++++++++++++++++++++++++++++++++++++ www/index.html | 4 ++++ www/js/main.js | 2 ++ 4 files changed, 97 insertions(+), 12 deletions(-) diff --git a/src/main.rs b/src/main.rs index 727ff45..7e23ac4 100644 --- a/src/main.rs +++ b/src/main.rs @@ -563,6 +563,13 @@ async fn main() -> Result<(), MainError> { .and(rss::with_rss_base_url(config.rss_base_url.clone())) .and_then(rss::lagging_nodes_response); + let unreachable_nodes_rss = warp::get() + .and(warp::path!("rss" / u32 / "unreachable.xml")) + .and(api::with_caches(caches.clone())) + .and(api::with_networks(network_infos.clone())) + .and(rss::with_rss_base_url(config.rss_base_url.clone())) + .and_then(rss::unreachable_nodes_response); + let networks_json = warp::get() .and(warp::path!("api" / "networks.json")) .and(api::with_networks(network_infos)) @@ -592,6 +599,7 @@ async fn main() -> Result<(), MainError> { .or(change_sse) .or(forks_rss) .or(lagging_nodes_rss) + .or(unreachable_nodes_rss) .or(invalid_blocks_rss); warp::serve(routes).run(config.address).await; @@ -636,14 +644,14 @@ mod tests { use crate::node::NodeInfo; async fn get_test_node_reachable(caches: &Caches, net_id: u32, node_id: u32) -> bool { - let locked_caches = caches.lock().await; - locked_caches - .get(&net_id) - .expect("network id should be there") - .node_data - .get(&node_id) - .expect("node id should be there") - .reachable + let locked_caches = caches.lock().await; + locked_caches + .get(&net_id) + .expect("network id should be there") + .node_data + .get(&node_id) + .expect("node id should be there") + .reachable } #[tokio::test] @@ -672,12 +680,21 @@ mod tests { }, ); } - assert_eq!(get_test_node_reachable(&caches, network_id, node.id).await, true); + assert_eq!( + get_test_node_reachable(&caches, network_id, node.id).await, + true + ); node_reachable(&caches, network_id, node.id, false).await; - assert_eq!(get_test_node_reachable(&caches, network_id, node.id).await, false); - + assert_eq!( + get_test_node_reachable(&caches, network_id, node.id).await, + false + ); + node_reachable(&caches, network_id, node.id, true).await; - assert_eq!(get_test_node_reachable(&caches, network_id, node.id).await, true); + assert_eq!( + get_test_node_reachable(&caches, network_id, node.id).await, + true + ); } } diff --git a/src/rss.rs b/src/rss.rs index f7f3d56..7221f2b 100644 --- a/src/rss.rs +++ b/src/rss.rs @@ -183,6 +183,17 @@ impl Item { guid: format!("lagging-node-{}-on-{}", node.name, height), } } + + pub fn unreachable_node_item(node: &NodeDataJson) -> Item { + Item { + title: format!("Node '{}' (id={}) is unreachable", node.name, node.id), + description: format!( + "The RPC server of this node is not reachable. The node might be offline or there might be other networking issues. The nodes tip data was last updated at timestamp {} (zero indicates never).", + node.last_changed_timestamp, + ), + guid: format!("unreachable-node-{}-last-{}", node.id, node.last_changed_timestamp), + } + } } pub async fn lagging_nodes_response( @@ -323,6 +334,57 @@ pub async fn invalid_blocks_response( } } +pub async fn unreachable_nodes_response( + network_id: u32, + caches: Caches, + network_infos: Vec, + base_url: String, +) -> Result { + let caches_locked = caches.lock().await; + + match caches_locked.get(&network_id) { + Some(cache) => { + let mut network_name = ""; + if let Some(network) = network_infos + .iter() + .filter(|net| net.id == network_id) + .collect::>() + .first() + { + network_name = &network.name; + } + + let unreachable_node_items: Vec = cache + .node_data + .values() + .filter(|node| !node.reachable) + .map(|node| Item::unreachable_node_item(node)) + .collect(); + let feed = Feed { + channel: Channel { + title: format!("Unreachable nodes - {}", network_name), + description: format!( + "Nodes on the {} network that can't be reached", + network_name + ), + link: format!( + "{}?network={}?src=unreachable-nodes", + base_url.clone(), + network_id + ), + href: format!("{}/rss/{}/unreachable.xml", base_url, network_id), + items: unreachable_node_items, + }, + }; + + return Ok(Response::builder() + .header("content-type", "application/rss+xml") + .body(feed.to_string())); + } + None => Ok(Ok(response_unknown_network(network_infos))), + } +} + pub fn response_unknown_network(network_infos: Vec) -> Response { let avaliable_networks = network_infos .iter() diff --git a/www/index.html b/www/index.html index 4dd9c9c..65ab927 100644 --- a/www/index.html +++ b/www/index.html @@ -49,6 +49,10 @@

Network:

Lagging nodes + + + Unreachable nodes +


diff --git a/www/js/main.js b/www/js/main.js index 7d2e514..251aa72 100644 --- a/www/js/main.js +++ b/www/js/main.js @@ -11,6 +11,7 @@ const connectionStatus = d3.select("#connection-status") const rssRecentForks = d3.select("#rss_recent_forks") const rssInvalidBlocks = d3.select("#rss_invalid_blocks") const rssLaggingNodes = d3.select("#rss_lagging_nodes") +const rssUnreachableNodes = d3.select("#rss_unreachable_nodes") const SEARCH_PARAM_NETWORK = "network" @@ -58,6 +59,7 @@ function update_network() { rssRecentForks.node().href = `rss/${current_network.id}/forks.xml` rssInvalidBlocks.node().href = `rss/${current_network.id}/invalid.xml` rssLaggingNodes.node().href = `rss/${current_network.id}/lagging.xml` + rssUnreachableNodes.node().href = `rss/${current_network.id}/unreachable.xml` } function set_initial_network() {