Gather memory usage statistics through parity-util-mem

cli/Cargo.toml

@@ -46,13 +46,6 @@ cli = [
 	"frame-benchmarking-cli",
 	"try-runtime-cli",
 	"polkadot-node-core-pvf",
-# memory stats require jemalloc, which we know is enabled for Linux
-# but not present on wasm or windows
-# https://github.com/paritytech/parity-common/blob/master/parity-util-mem/src/allocators.rs#L9-L34
-# Once
-# https://github.com/rust-lang/cargo/issues/1197
-# is resolved.
-	"service/memory-stats",
 ]
 runtime-benchmarks = [ "service/runtime-benchmarks" ]
 trie-memory-tracker = [ "sp-trie/memory-tracker" ]

node/metrics/Cargo.toml

@@ -13,8 +13,5 @@ metered-channel = { path = "../metered-channel" }
 
 substrate-prometheus-endpoint = { git = "https://github.com/paritytech/substrate", branch = "master" }
 
-jemalloc-ctl = { version = "0.3.3", optional = true }
-
 [features]
 default = []
-memory-stats = ["jemalloc-ctl"]

node/metrics/src/lib.rs

@@ -26,13 +26,6 @@
 
 pub use metered_channel as metered;
 
-/// Memory allocation stats tracking.
-#[cfg(feature = "memory-stats")]
-pub mod memory_stats;
-
-#[cfg(feature = "memory-stats")]
-pub use self::memory_stats::{MemoryAllocationSnapshot, MemoryAllocationTracker};
-
 /// Cyclic metric collection support.
 pub mod metronome;
 pub use self::metronome::Metronome;

node/overseer/Cargo.toml

@@ -19,6 +19,7 @@ polkadot-overseer-gen = { path = "./overseer-gen" }
 polkadot-overseer-all-subsystems-gen = { path = "./all-subsystems-gen" }
 tracing = "0.1.27"
 lru = "0.6"
+parity-util-mem = { version = ">= 0.10.1", default-features = false }
 
 [dev-dependencies]
 metered-channel = { path = "../metered-channel" }
@@ -29,4 +30,3 @@ assert_matches = "1.4.0"
 
 [features]
 default = []
-memory-stats = ["polkadot-node-metrics/memory-stats"]

node/overseer/src/lib.rs

@@ -108,8 +108,7 @@ use polkadot_node_metrics::{
 	Metronome,
 };
 
-#[cfg(feature = "memory-stats")]
-use polkadot_node_metrics::memory_stats::MemoryAllocationTracker;
+use parity_util_mem::MemoryAllocationTracker;
 
 pub use polkadot_overseer_gen as gen;
 pub use polkadot_overseer_gen::{
@@ -649,28 +648,39 @@ where
 			}
 			let subsystem_meters = overseer.map_subsystems(ExtractNameAndMeters);
 
-			#[cfg(feature = "memory-stats")]
-			let memory_stats = MemoryAllocationTracker::new().expect("Jemalloc is the default allocator. qed");
+			let memory_stats = match MemoryAllocationTracker::new() {
+				Ok(memory_stats) => Some(memory_stats),
+				Err(error) => {
+					tracing::debug!(
+						target: LOG_TARGET,
+						"Failed to initialize memory allocation tracker: {:?}",
+						error
+					);
+
+					None
+				},
+			};
 
 			let metronome_metrics = metrics.clone();
 			let metronome =
 				Metronome::new(std::time::Duration::from_millis(950)).for_each(move |_| {
-					#[cfg(feature = "memory-stats")]
-					match memory_stats.snapshot() {
-						Ok(memory_stats_snapshot) => {
-							tracing::trace!(
+					if let Some(ref memory_stats) = memory_stats {
+						match memory_stats.snapshot() {
+							Ok(memory_stats_snapshot) => {
+								tracing::trace!(
+									target: LOG_TARGET,
+									"memory_stats: {:?}",
+									&memory_stats_snapshot
+								);
+								metronome_metrics.memory_stats_snapshot(memory_stats_snapshot);
+							},
+
+							Err(e) => tracing::debug!(
 								target: LOG_TARGET,
-								"memory_stats: {:?}",
-								&memory_stats_snapshot
-							);
-							metronome_metrics.memory_stats_snapshot(memory_stats_snapshot);
-						},
-
-						Err(e) => tracing::debug!(
-							target: LOG_TARGET,
-							"Failed to obtain memory stats: {:?}",
-							e
-						),
+								"Failed to obtain memory stats: {:?}",
+								e
+							),
+						}
 					}
 
 					// We combine the amount of messages from subsystems to the overseer

node/overseer/src/metrics.rs

@@ -19,8 +19,7 @@
 use super::*;
 pub use polkadot_node_metrics::metrics::{self, prometheus, Metrics as MetricsTrait};
 
-#[cfg(feature = "memory-stats")]
-use polkadot_node_metrics::MemoryAllocationSnapshot;
+use parity_util_mem::MemoryAllocationSnapshot;
 
 /// Overseer Prometheus metrics.
 #[derive(Clone)]
@@ -35,10 +34,7 @@ struct MetricsInner {
 	signals_sent: prometheus::GaugeVec<prometheus::U64>,
 	signals_received: prometheus::GaugeVec<prometheus::U64>,
 
-	#[cfg(feature = "memory-stats")]
 	memory_stats_resident: prometheus::Gauge<prometheus::U64>,
-
-	#[cfg(feature = "memory-stats")]
 	memory_stats_allocated: prometheus::Gauge<prometheus::U64>,
 }
 
@@ -65,13 +61,10 @@ impl Metrics {
 		}
 	}
 
-	#[cfg(feature = "memory-stats")]
 	pub(crate) fn memory_stats_snapshot(&self, memory_stats: MemoryAllocationSnapshot) {
 		if let Some(metrics) = &self.0 {
-			let MemoryAllocationSnapshot { resident, allocated } = memory_stats;
-
-			metrics.memory_stats_allocated.set(allocated);
-			metrics.memory_stats_resident.set(resident);
+			metrics.memory_stats_allocated.set(memory_stats.allocated);
+			metrics.memory_stats_resident.set(memory_stats.resident);
 		}
 	}
 
@@ -202,16 +195,13 @@ impl MetricsTrait for Metrics {
 				registry,
 			)?,
 
-			#[cfg(feature = "memory-stats")]
 			memory_stats_allocated: prometheus::register(
 				prometheus::Gauge::<prometheus::U64>::new(
 					"memory_allocated",
 					"Total bytes allocated by the node",
 				)?,
 				registry,
 			)?,
-
-			#[cfg(feature = "memory-stats")]
 			memory_stats_resident: prometheus::register(
 				prometheus::Gauge::<prometheus::U64>::new(
 					"memory_resident",

node/service/Cargo.toml

@@ -175,5 +175,4 @@ try-runtime = [
 	"rococo-runtime/try-runtime",
 ]
 malus = ["full-node"]
-memory-stats = ["polkadot-overseer/memory-stats"]
 disputes = ["polkadot-node-core-dispute-coordinator/disputes"]