percentile math

block_priofees/Cargo.toml

@@ -11,6 +11,7 @@ solana-sdk = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
 log = { workspace = true }
+itertools = { workspace = true }
 dashmap = { workspace = true }
 jsonrpsee = { workspace = true }
 tracing-subscriber = { workspace = true }

block_priofees/src/block_priofees.rs

@@ -7,6 +7,7 @@ use std::sync::Arc;
 use tokio::sync::broadcast::error::RecvError::{Closed, Lagged};
 use tokio::sync::broadcast::Sender;
 use tokio::task::JoinHandle;
+use crate::stats_calculation::calculate_supp_stats;
 
 // note: ATM only the latest slot (highest key) is used
 const SLOTS_TO_RETAIN: u64 = 5000;
@@ -70,7 +71,7 @@ pub async fn start_block_priofees_task(
                         })
                         .collect::<Vec<(u64, u64)>>();
 
-                    let priofees_stats = calculate_supp_info(&block_prio_fees);
+                    let priofees_stats = calculate_supp_stats(&block_prio_fees);
 
                     trace!("Got prio fees stats for confirmed block {}", confirmed_slot);
 
@@ -115,49 +116,3 @@ pub async fn start_block_priofees_task(
     )
 }
 
-
-fn calculate_supp_info(
-    // Vec(prioritization_fees, cu_consumed)
-    prio_fees_in_block: &Vec<(u64, u64)>,
-) -> PrioFeesStats {
-    let mut prio_fees_in_block = if prio_fees_in_block.is_empty() {
-        // TODO is that smart?
-        vec![(0, 0)]
-    } else {
-        prio_fees_in_block.clone()
-    };
-    prio_fees_in_block.sort_by(|a, b| a.0.cmp(&b.0));
-
-    let median_index = prio_fees_in_block.len() / 2;
-    let p75_index = prio_fees_in_block.len() * 75 / 100;
-    let p90_index = prio_fees_in_block.len() * 90 / 100;
-    let p_min = prio_fees_in_block[0].0;
-    let p_median = prio_fees_in_block[median_index].0;
-    let p_75 = prio_fees_in_block[p75_index].0;
-    let p_90 = prio_fees_in_block[p90_index].0;
-    let p_max = prio_fees_in_block.last().map(|x| x.0).unwrap();
-
-    PrioFeesStats {
-        p_min,
-        p_median,
-        p_75,
-        p_90,
-        p_max,
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn test_calculate_supp_info() {
-        let prio_fees_in_block = vec![(2, 2), (4, 4), (5, 5), (3, 3), (1, 1)];
-        let supp_info = calculate_supp_info(&prio_fees_in_block);
-        assert_eq!(supp_info.p_min, 1);
-        assert_eq!(supp_info.p_median, 3);
-        assert_eq!(supp_info.p_75, 4);
-        assert_eq!(supp_info.p_90, 5);
-        assert_eq!(supp_info.p_max, 5);
-    }
-}

block_priofees/src/lib.rs

@@ -1,4 +1,6 @@
 mod block_priofees;
 
 pub mod rpc_data;
+mod stats_calculation;
+
 pub use block_priofees::{start_block_priofees_task, PrioFeesService};

block_priofees/src/rpc_data.rs

@@ -1,3 +1,4 @@
+use std::collections::HashMap;
 use jsonrpsee::core::Serialize;
 use solana_sdk::clock::Slot;
 
@@ -8,6 +9,8 @@ pub struct PrioFeesStats {
     pub p_75: u64,
     pub p_90: u64,
     pub p_max: u64,
+    // p0, p5, p10, ..., p95, p100
+    pub fine_percentiles: HashMap<String, u64>,
 }
 
 

block_priofees/src/stats_calculation.rs

@@ -0,0 +1,97 @@
+use std::collections::HashMap;
+use itertools::Itertools;
+use crate::rpc_data::PrioFeesStats;
+
+pub fn calculate_supp_stats(
+    // Vec(prioritization_fees, cu_consumed)
+    prio_fees_in_block: &Vec<(u64, u64)>,
+) -> PrioFeesStats {
+    let mut prio_fees_in_block = if prio_fees_in_block.is_empty() {
+        // TODO is that smart?
+        vec![(0, 0)]
+    } else {
+        prio_fees_in_block.clone()
+    };
+    prio_fees_in_block.sort_by(|a, b| a.0.cmp(&b.0));
+
+    let median_index = prio_fees_in_block.len() / 2;
+    let p75_index = prio_fees_in_block.len() * 75 / 100;
+    let p90_index = prio_fees_in_block.len() * 90 / 100;
+    let p_min = prio_fees_in_block[0].0;
+    let p_median = prio_fees_in_block[median_index].0;
+    let p_75 = prio_fees_in_block[p75_index].0;
+    let p_90 = prio_fees_in_block[p90_index].0;
+    let p_max = prio_fees_in_block.last().map(|x| x.0).unwrap();
+
+    let fine_percentiles: HashMap<String, u64> =
+        (0..=100).step_by(5).map(|percent| percent).map(|x| {
+            let prio_fee = if x == 100 {
+                prio_fees_in_block.last().unwrap().0
+            } else {
+                let index = prio_fees_in_block.len() * x / 100;
+                prio_fees_in_block[index].0
+            };
+            (format!("p{}", x), prio_fee)
+        }).into_group_map_by(|x| x.0.clone())
+            .into_iter()
+        .map(|(k, v)| (k, v.iter().exactly_one().unwrap().1))
+        .collect();
+
+    assert_eq!(p_min, *fine_percentiles.get("p0").unwrap());
+    assert_eq!(p_median, *fine_percentiles.get("p50").unwrap());
+    assert_eq!(p_75, *fine_percentiles.get("p75").unwrap());
+    assert_eq!(p_90, *fine_percentiles.get("p90").unwrap());
+    assert_eq!(p_max, *fine_percentiles.get("p100").unwrap());
+
+    PrioFeesStats {
+        p_min,
+        p_median,
+        p_75,
+        p_90,
+        p_max,
+        fine_percentiles,
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_calculate_supp_info() {
+        let prio_fees_in_block = vec![(2, 2), (4, 4), (5, 5), (3, 3), (1, 1)];
+        let supp_info = calculate_supp_stats(&prio_fees_in_block);
+        assert_eq!(supp_info.p_min, 1);
+        assert_eq!(supp_info.p_median, 3);
+        assert_eq!(supp_info.p_75, 4);
+        assert_eq!(supp_info.p_90, 5);
+        assert_eq!(supp_info.p_max, 5);
+    }
+
+    #[test]
+    fn test_statisticshowto() {
+        let prio_fees_in_block = vec![
+            (30, 1),
+            (33, 2),
+            (43, 3),
+            (53, 4),
+            (56, 5),
+            (67, 6),
+            (68, 7),
+            (72, 8),
+            ];
+        let supp_info = calculate_supp_stats(&prio_fees_in_block);
+        assert_eq!(supp_info.fine_percentiles.get("p25").unwrap(), &43);
+    }
+
+
+    #[test]
+    fn test_large_list() {
+        let prio_fees_in_block: Vec<(u64,u64)> = (0..1000).map(|x| (x,x)).collect();
+        let supp_info = calculate_supp_stats(&prio_fees_in_block);
+        assert_eq!(supp_info.fine_percentiles.get("p95").unwrap(), &950);
+
+    }
+
+
+}