From ac1170350e900c04e95d7a44a7d409eb352a3f85 Mon Sep 17 00:00:00 2001 From: Brian Anderson Date: Wed, 20 Mar 2019 20:20:18 -0700 Subject: [PATCH] tikv_alloc: abstract some new jemalloc code (#4411) * tikv_alloc: abstract some new jemalloc code This renames the jealloc_metrics module to allocator_metrics, does similar for the code inside it. It implements a no-op fetch_stats method for the system allocator. It marks the jemalloc-sys and jemalloc-ctl crates as optional, so that they might not be compiled when jemalloc is disabled. It's not actually possible disable the jemalloc features in this PR because there is no way to turn off the default-features of the tikv_alloc crate. A followup in tikv#4207 will make it possible. It also moves a `mod` to after some `use` statements. Signed-off-by: Brian Anderson --- components/tikv_alloc/Cargo.toml | 11 ++++- components/tikv_alloc/src/lib.rs | 38 ++++++++------- src/bin/util/setup.rs | 4 +- src/util/metrics/allocator_metrics.rs | 55 +++++++++++++++++++++ src/util/metrics/jemalloc_metrics.rs | 70 --------------------------- src/util/metrics/mod.rs | 7 ++- 6 files changed, 90 insertions(+), 95 deletions(-) create mode 100644 src/util/metrics/allocator_metrics.rs delete mode 100644 src/util/metrics/jemalloc_metrics.rs diff --git a/components/tikv_alloc/Cargo.toml b/components/tikv_alloc/Cargo.toml index a80f7760ea6..117121abdeb 100644 --- a/components/tikv_alloc/Cargo.toml +++ b/components/tikv_alloc/Cargo.toml @@ -6,7 +6,8 @@ authors = ["Brian Anderson "] publish = false [features] -default = ["jemallocator"] +default = ["jemalloc"] +jemalloc = ["jemallocator", "jemalloc-sys", "jemalloc-ctl"] # Build jemalloc's profiling features. Without this # certain profile functions will return nothing. @@ -27,9 +28,17 @@ version = "0.1.9" optional = true features = ["unprefixed_malloc_on_supported_platforms"] +# FIXME: shouldn't need to link to this crate, but the 'stats' +# feature is missing in jemallocator. +# +# https://github.com/gnzlbg/jemallocator/issues/112 +# https://github.com/tikv/tikv/issues/4406 + [dependencies.jemalloc-sys] version = "0.1.8" +optional = true features = ["stats"] [dependencies.jemalloc-ctl] version = "0.2.0" +optional = true diff --git a/components/tikv_alloc/src/lib.rs b/components/tikv_alloc/src/lib.rs index 4492d37bc66..ed2e0f24456 100644 --- a/components/tikv_alloc/src/lib.rs +++ b/components/tikv_alloc/src/lib.rs @@ -106,9 +106,12 @@ static ALLOC: std::alloc::System = std::alloc::System; pub use self::imp::*; +pub type AllocStats = Vec<(&'static str, usize)>; + // The implementation of this crate when jemalloc is turned on #[cfg(all(unix, not(fuzzing), not(feature = "no-jemalloc")))] mod imp { + use super::AllocStats; use jemalloc_ctl::{stats, Epoch as JeEpoch}; use jemallocator::ffi::malloc_stats_print; use libc::{self, c_char, c_void}; @@ -128,27 +131,18 @@ mod imp { String::from_utf8_lossy(&buf).into_owned() } - pub struct JemallocStats { - pub allocated: usize, - pub active: usize, - pub metadata: usize, - pub resident: usize, - pub mapped: usize, - pub retained: usize, - } - - pub fn fetch_stats() -> io::Result { + pub fn fetch_stats() -> io::Result> { // Stats are cached. Need to advance epoch to refresh. JeEpoch::new()?.advance()?; - Ok(JemallocStats { - allocated: stats::allocated()?, - active: stats::active()?, - metadata: stats::metadata()?, - resident: stats::resident()?, - mapped: stats::mapped()?, - retained: stats::retained()?, - }) + Ok(Some(vec![ + ("allocated", stats::allocated()?), + ("active", stats::active()?), + ("metadata", stats::metadata()?), + ("resident", stats::resident()?), + ("mapped", stats::mapped()?), + ("retained", stats::retained()?), + ])) } #[allow(clippy::cast_ptr_alignment)] @@ -304,8 +298,16 @@ mod imp { #[cfg(not(all(unix, not(fuzzing), not(feature = "no-jemalloc"))))] mod imp { + + use super::AllocStats; + use std::io; + pub fn dump_stats() -> String { String::new() } pub fn dump_prof(_path: Option<&str>) {} + + pub fn fetch_stats() -> io::Result> { + Ok(None) + } } diff --git a/src/bin/util/setup.rs b/src/bin/util/setup.rs index d22209a7e06..a12d20fdc46 100644 --- a/src/bin/util/setup.rs +++ b/src/bin/util/setup.rs @@ -68,8 +68,8 @@ pub fn initial_logger(config: &TiKvConfig) { pub fn initial_metric(cfg: &MetricConfig, node_id: Option) { util::metrics::monitor_threads("tikv") .unwrap_or_else(|e| fatal!("failed to start monitor thread: {}", e)); - util::metrics::monitor_jemalloc_stats("tikv") - .unwrap_or_else(|e| fatal!("failed to monitor jemalloc stats: {}", e)); + util::metrics::monitor_allocator_stats("tikv") + .unwrap_or_else(|e| fatal!("failed to monitor allocator stats: {}", e)); if cfg.interval.as_secs() == 0 || cfg.address.is_empty() { return; diff --git a/src/util/metrics/allocator_metrics.rs b/src/util/metrics/allocator_metrics.rs new file mode 100644 index 00000000000..e4f5f12df23 --- /dev/null +++ b/src/util/metrics/allocator_metrics.rs @@ -0,0 +1,55 @@ +// Copyright 2017 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +use prometheus::core::{Collector, Desc}; +use prometheus::proto::MetricFamily; +use prometheus::{IntGaugeVec, Opts, Result}; + +use tikv_alloc; + +pub fn monitor_allocator_stats>(namespace: S) -> Result<()> { + prometheus::register(Box::new(AllocStatsCollector::new(namespace)?)) +} + +struct AllocStatsCollector { + descs: Vec, + metrics: IntGaugeVec, +} + +impl AllocStatsCollector { + fn new>(namespace: S) -> Result { + let stats = IntGaugeVec::new( + Opts::new("allocator_stats", "Allocator stats").namespace(namespace.into()), + &["type"], + )?; + Ok(AllocStatsCollector { + descs: stats.desc().into_iter().cloned().collect(), + metrics: stats, + }) + } +} + +impl Collector for AllocStatsCollector { + fn desc(&self) -> Vec<&Desc> { + self.descs.iter().collect() + } + + fn collect(&self) -> Vec { + if let Ok(Some(stats)) = tikv_alloc::fetch_stats() { + for stat in stats { + self.metrics.with_label_values(&[stat.0]).set(stat.1 as i64); + } + } + self.metrics.collect() + } +} diff --git a/src/util/metrics/jemalloc_metrics.rs b/src/util/metrics/jemalloc_metrics.rs deleted file mode 100644 index e09d2ee0d85..00000000000 --- a/src/util/metrics/jemalloc_metrics.rs +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright 2017 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -use prometheus::core::{Collector, Desc}; -use prometheus::proto::MetricFamily; -use prometheus::{IntGaugeVec, Opts, Result}; - -use tikv_alloc; - -pub fn monitor_jemalloc_stats>(namespace: S) -> Result<()> { - prometheus::register(Box::new(JemallocStatsCollector::new(namespace)?)) -} - -struct JemallocStatsCollector { - descs: Vec, - metrics: IntGaugeVec, -} - -impl JemallocStatsCollector { - fn new>(namespace: S) -> Result { - let stats = IntGaugeVec::new( - Opts::new("jemalloc_stats", "Jemalloc stats").namespace(namespace.into()), - &["type"], - )?; - Ok(JemallocStatsCollector { - descs: stats.desc().into_iter().cloned().collect(), - metrics: stats, - }) - } -} - -impl Collector for JemallocStatsCollector { - fn desc(&self) -> Vec<&Desc> { - self.descs.iter().collect() - } - - fn collect(&self) -> Vec { - if let Ok(stats) = tikv_alloc::fetch_stats() { - self.metrics - .with_label_values(&["allocated"]) - .set(stats.allocated as i64); - self.metrics - .with_label_values(&["active"]) - .set(stats.active as i64); - self.metrics - .with_label_values(&["metadata"]) - .set(stats.metadata as i64); - self.metrics - .with_label_values(&["resident"]) - .set(stats.resident as i64); - self.metrics - .with_label_values(&["mapped"]) - .set(stats.mapped as i64); - self.metrics - .with_label_values(&["retained"]) - .set(stats.retained as i64); - } - self.metrics.collect() - } -} diff --git a/src/util/metrics/mod.rs b/src/util/metrics/mod.rs index 1c78200bfe7..1cbb8ba90a0 100644 --- a/src/util/metrics/mod.rs +++ b/src/util/metrics/mod.rs @@ -11,8 +11,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -pub mod jemalloc_metrics; - use std::thread; use std::time::Duration; @@ -28,8 +26,9 @@ mod threads_dummy; #[cfg(not(target_os = "linux"))] pub use self::threads_dummy::monitor_threads; -#[cfg(all(unix, not(fuzzing), not(feature = "no-jemalloc")))] -pub use self::jemalloc_metrics::monitor_jemalloc_stats; +pub use self::allocator_metrics::monitor_allocator_stats; + +pub mod allocator_metrics; /// Runs a background Prometheus client. pub fn run_prometheus(