fix(discv5): update metrics wrt unverifiable enrs

crates/net/discv5/src/lib.rs

@@ -273,6 +273,8 @@ impl Discv5 {
                     "discovered unverifiable enr, source socket doesn't match socket advertised in ENR"
                 );
 
+                self.metrics.discovered_peers.increment_unverifiable_enrs_raw_total(1);
+
                 self.on_discovered_peer(&enr, socket)
             }
             _ => None

crates/net/discv5/src/metrics.rs

@@ -21,66 +21,80 @@ pub struct DiscoveredPeersMetrics {
     // Kbuckets
     ////////////////////////////////////////////////////////////////////////////////////////////////
     /// Total peers currently in [`discv5::Discv5`]'s kbuckets.
-    total_kbucket_peers_raw: Gauge,
+    kbucket_peers_raw_total: Gauge,
     /// Total discovered peers that are inserted into [`discv5::Discv5`]'s kbuckets.
     ///
     /// This is a subset of the total established sessions, in which all peers advertise a udp
     /// socket in their node record which is reachable from the local node. Only these peers make
     /// it into [`discv5::Discv5`]'s kbuckets and will hence be included in queries.
     ///
     /// Note: the definition of 'discovered' is not exactly synonymous in `reth_discv4::Discv4`.
-    total_inserted_kbucket_peers_raw: Counter,
+    inserted_kbucket_peers_raw_total: Counter,
 
     ////////////////////////////////////////////////////////////////////////////////////////////////
     // Sessions
     ////////////////////////////////////////////////////////////////////////////////////////////////
     /// Total peers currently connected to [`discv5::Discv5`].
-    total_sessions_raw: Gauge,
+    sessions_raw_total: Gauge,
     /// Total number of sessions established by [`discv5::Discv5`].
-    total_established_sessions_raw: Counter,
+    established_sessions_raw_total: Counter,
     /// Total number of sessions established by [`discv5::Discv5`], with peers that don't advertise
     /// a socket which is reachable from the local node in their node record.
     ///
     /// These peers can't make it into [`discv5::Discv5`]'s kbuckets, and hence won't be part of
     /// queries (neither shared with peers in NODES responses, nor queried for peers with FINDNODE
     /// requests).
-    total_established_sessions_unreachable_enr: Counter,
+    established_sessions_unreachable_enr_total: Counter,
     /// Total number of sessions established by [`discv5::Discv5`], that pass configured
     /// [`filter`](crate::filter) rules.
-    total_established_sessions_custom_filtered: Counter,
+    established_sessions_custom_filtered_total: Counter,
+    /// Total number of unverifiable ENRs discovered by [`discv5::Discv5`].
+    ///
+    /// These are peers that fail [`discv5::Discv5`] session establishment, because the UDP socket
+    /// they're making a connection from doesn't match the UDP socket advertised in their ENR.
+    /// These peers will be denied a session (and hence can't make it into kbuckets) until they
+    /// have update their ENR, to reflect their actual UDP socket.
+    unverifiable_enrs_raw_total: Counter,
 }
 
 impl DiscoveredPeersMetrics {
     /// Sets current total number of peers in [`discv5::Discv5`]'s kbuckets.
     pub fn set_total_kbucket_peers(&self, num: usize) {
-        self.total_kbucket_peers_raw.set(num as f64)
+        self.kbucket_peers_raw_total.set(num as f64)
     }
 
     /// Increments the number of kbucket insertions in [`discv5::Discv5`].
     pub fn increment_kbucket_insertions(&self, num: u64) {
-        self.total_inserted_kbucket_peers_raw.increment(num)
+        self.inserted_kbucket_peers_raw_total.increment(num)
     }
 
     /// Sets current total number of peers connected to [`discv5::Discv5`].
     pub fn set_total_sessions(&self, num: usize) {
-        self.total_sessions_raw.set(num as f64)
+        self.sessions_raw_total.set(num as f64)
     }
 
     /// Increments number of sessions established by [`discv5::Discv5`].
     pub fn increment_established_sessions_raw(&self, num: u64) {
-        self.total_established_sessions_raw.increment(num)
+        self.established_sessions_raw_total.increment(num)
     }
 
     /// Increments number of sessions established by [`discv5::Discv5`], with peers that don't have
     /// a reachable node record.
     pub fn increment_established_sessions_unreachable_enr(&self, num: u64) {
-        self.total_established_sessions_unreachable_enr.increment(num)
+        self.established_sessions_unreachable_enr_total.increment(num)
     }
 
     /// Increments number of sessions established by [`discv5::Discv5`], that pass configured
     /// [`filter`](crate::filter) rules.
     pub fn increment_established_sessions_filtered(&self, num: u64) {
-        self.total_established_sessions_custom_filtered.increment(num)
+        self.established_sessions_custom_filtered_total.increment(num)
+    }
+
+    /// Increments number of unverifiable ENRs discovered by [`discv5::Discv5`]. These are peers
+    /// that fail session establishment because their advertised UDP socket doesn't match the
+    /// socket they are making the connection from.
+    pub fn increment_unverifiable_enrs_raw_total(&self, num: u64) {
+        self.unverifiable_enrs_raw_total.increment(num)
     }
 }
 

etc/grafana/dashboards/reth-discovery.json

@@ -52,6 +52,7 @@
       }
     ]
   },
+  "description": "Devp2p peer discovery protocols",
   "editable": true,
   "fiscalYearStartMonth": 0,
   "graphTooltip": 0,
@@ -578,7 +579,7 @@
               {
                 "id": "color",
                 "value": {
-                  "fixedColor": "purple",
+                  "fixedColor": "#9b73d6",
                   "mode": "fixed"
                 }
               }
@@ -628,7 +629,7 @@
           },
           "disableTextWrap": false,
           "editorMode": "builder",
-          "expr": "reth_discv5_total_kbucket_peers_raw{instance=\"$instance\"}",
+          "expr": "reth_discv5_kbucket_peers_raw_total{instance=\"$instance\"}",
           "fullMetaSearch": false,
           "includeNullMetadata": true,
           "instant": false,
@@ -644,7 +645,7 @@
           },
           "disableTextWrap": false,
           "editorMode": "builder",
-          "expr": "reth_discv5_total_sessions_raw{instance=\"$instance\"}",
+          "expr": "reth_discv5_sessions_raw_total{instance=\"$instance\"}",
           "fullMetaSearch": false,
           "hide": false,
           "includeNullMetadata": true,
@@ -663,7 +664,7 @@
         "type": "prometheus",
         "uid": "${DS_PROMETHEUS}"
       },
-      "description": "Frequency of session establishment and kbuckets insertions.\n\nSince discv5 favours long-lived connections, kbuckets insertions are expected to be less frequent the longer the node stays online.\n\nSome incoming connections may be from peers with unreachable ENRs, ENRs that don't advertise a  UDP socket. These peers are not useful for the discv5 node, nor for RLPx.\n\nDiscovered peers are filtered w.r.t. what they advertise in their ENR. By default peers advertising 'eth2' are filtered out. Unreachable ENRs are also filtered out. Only peers that pass the filter are useful. These peers get passed up the node, to attempt an RLPx connection.\n\n",
+      "description": "Frequency of session establishment and kbuckets insertions.\n\nSince discv5 favours long-lived connections, kbuckets insertions are expected to be less frequent the longer the node stays online.\n\nSome incoming connections may be from peers with unreachable ENRs, ENRs that don't advertise a  UDP socket. These peers are not useful for the discv5 node, nor for RLPx.\n\nDiscovered peers are filtered w.r.t. what they advertise in their ENR. By default peers advertising 'eth2' are filtered out. Unreachable ENRs are also filtered out. Only peers that pass the filter are useful. These peers get passed up the node, to attempt an RLPx connection.\n\nSessions will succeed to peers that advertise no UDP socket in their ENR. This allows peers to discover their reachable socket. On the other hand, for DoS protection, peers that advertise a different socket than the socket from which they make the connection, are denied a sigp/discv5 session. These peers have an unverifiable ENR. The peers are passed to RLPx nonetheless (some EL implementations of discv5 are more lax about ENR and source socket matching). ",
       "fieldConfig": {
         "defaults": {
           "color": {
@@ -728,7 +729,7 @@
               {
                 "id": "color",
                 "value": {
-                  "fixedColor": "purple",
+                  "fixedColor": "light-green",
                   "mode": "fixed"
                 }
               }
@@ -743,7 +744,7 @@
               {
                 "id": "color",
                 "value": {
-                  "fixedColor": "super-light-red",
+                  "fixedColor": "#9958f4",
                   "mode": "fixed"
                 }
               }
@@ -778,6 +779,36 @@
                 }
               }
             ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Session Establishment Failed (unverifiable ENR)"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "#8ab8ff",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Failed Session Establishments (unverifiable ENR)"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "#8ab8ff",
+                  "mode": "fixed"
+                }
+              }
+            ]
           }
         ]
       },
@@ -808,7 +839,7 @@
           },
           "disableTextWrap": false,
           "editorMode": "builder",
-          "expr": "rate(reth_discv5_total_inserted_kbucket_peers_raw{instance=\"$instance\"}[$__rate_interval])",
+          "expr": "rate(reth_discv5_inserted_kbucket_peers_raw_total{instance=\"$instance\"}[$__rate_interval])",
           "fullMetaSearch": false,
           "includeNullMetadata": false,
           "instant": false,
@@ -824,7 +855,7 @@
           },
           "disableTextWrap": false,
           "editorMode": "builder",
-          "expr": "rate(reth_discv5_total_established_sessions_raw{instance=\"$instance\"}[$__rate_interval])",
+          "expr": "rate(reth_discv5_established_sessions_raw_total{instance=\"$instance\"}[$__rate_interval])",
           "fullMetaSearch": false,
           "hide": false,
           "includeNullMetadata": false,
@@ -841,7 +872,7 @@
           },
           "disableTextWrap": false,
           "editorMode": "builder",
-          "expr": "rate(reth_discv5_total_established_sessions_unreachable_enr{instance=\"$instance\"}[$__rate_interval])",
+          "expr": "rate(reth_discv5_established_sessions_unreachable_enr_total{instance=\"$instance\"}[$__rate_interval])",
           "fullMetaSearch": false,
           "hide": false,
           "includeNullMetadata": false,
@@ -858,7 +889,7 @@
           },
           "disableTextWrap": false,
           "editorMode": "builder",
-          "expr": "rate(reth_discv5_total_established_sessions_raw{instance=\"$instance\"}[$__rate_interval]) - rate(reth_discv5_total_established_sessions_custom_filtered{instance=\"$instance\"}[$__rate_interval])",
+          "expr": "rate(reth_discv5_established_sessions_raw_total{instance=\"$instance\"}[$__rate_interval]) - rate(reth_discv5_established_sessions_custom_filtered_total{instance=\"$instance\"}[$__rate_interval])",
           "fullMetaSearch": false,
           "hide": false,
           "includeNullMetadata": false,
@@ -867,6 +898,23 @@
           "range": true,
           "refId": "D",
           "useBackend": false
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "disableTextWrap": false,
+          "editorMode": "builder",
+          "expr": "rate(reth_discv5_unverifiable_enrs_raw_total{instance=\"$instance\"}[$__rate_interval])",
+          "fullMetaSearch": false,
+          "hide": false,
+          "includeNullMetadata": true,
+          "instant": false,
+          "legendFormat": "Failed Session Establishments (unverifiable ENR)",
+          "range": true,
+          "refId": "E",
+          "useBackend": false
         }
       ],
       "title": "Peer Churn",
@@ -942,7 +990,7 @@
               {
                 "id": "color",
                 "value": {
-                  "fixedColor": "purple",
+                  "fixedColor": "#b677d9",
                   "mode": "fixed"
                 }
               }
@@ -957,7 +1005,7 @@
               {
                 "id": "color",
                 "value": {
-                  "fixedColor": "super-light-red",
+                  "fixedColor": "light-green",
                   "mode": "fixed"
                 }
               }
@@ -977,6 +1025,21 @@
                 }
               }
             ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "OP EL"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "#8AB8FF",
+                  "mode": "fixed"
+                }
+              }
+            ]
           }
         ]
       },
@@ -1074,7 +1137,7 @@
           },
           "disableTextWrap": false,
           "editorMode": "code",
-          "expr": "rate(reth_discv5_total_established_sessions_raw{instance=\"$instance\"}[$__rate_interval]) - (rate(reth_discv5_eth{instance=\"$instance\"}[$__rate_interval]) + rate(reth_discv5_eth2{instance=\"$instance\"}[$__rate_interval]) + rate(reth_discv5_opstack{instance=\"$instance\"}[$__rate_interval]) + rate(reth_discv5_opel{instance=\"$instance\"}[$__rate_interval]))",
+          "expr": "(rate(reth_discv5_established_sessions_raw_total{instance=\"$instance\"}[$__rate_interval]) + rate(reth_discv5_unverifiable_enrs_raw_total{instance=\"$instance\"}[$__rate_interval])) - (rate(reth_discv5_eth{instance=\"$instance\"}[$__rate_interval]) + rate(reth_discv5_eth2{instance=\"$instance\"}[$__rate_interval]) + rate(reth_discv5_opstack{instance=\"$instance\"}[$__rate_interval]) + rate(reth_discv5_opel{instance=\"$instance\"}[$__rate_interval]))",
           "fullMetaSearch": false,
           "hide": false,
           "includeNullMetadata": false,
@@ -1125,7 +1188,7 @@
   "timepicker": {},
   "timezone": "",
   "title": "Reth - Peer Discovery",
-  "uid": "de6e87b2-7630-40b2-b2c4-a500476e799d",
+  "uid": "fd2d69b5-ca32-45d0-946e-c00ddcd7052c",
   "version": 1,
   "weekStart": ""
 }

etc/grafana/dashboards/reth-state-growth.json

@@ -1652,6 +1652,28 @@
     "tags": [],
     "templating": {
       "list": [
+        {
+          "current": {},
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "definition": "query_result(reth_info)",
+          "hide": 0,
+          "includeAll": false,
+          "multi": false,
+          "name": "instance",
+          "options": [],
+          "query": {
+            "query": "query_result(reth_info)",
+            "refId": "PrometheusVariableQueryEditor-VariableQuery"
+          },
+          "refresh": 1,
+          "regex": "/.*instance=\\\"([^\\\"]*).*/",
+          "skipUrlSync": false,
+          "sort": 0,
+          "type": "query"
+        },
         {
           "current": {
             "selected": true,