Skip to content

Commit

Permalink
Add prometheus publishing and hook up FilesystemStore
Browse files Browse the repository at this point in the history 
Defines the framework for publishing metrics through prometheus
and seed the implementation with stats on FilesystemStore and
EvictionMap.
  • Loading branch information
@allada
allada committed Jul 18, 2023
1 parent 4856d35 commit 04a7772
Show file tree
Hide file tree
Showing 21 changed files with 752 additions and 28 deletions.
163 changes: 162 additions & 1 deletion Cargo.Bazel.lock
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
{
"checksum": "ede2ad6fc879a28275ce174d5f4e7010b13c85a51f7d70a742a4b8a88639dcb2",
"checksum": "eebd7d004f166798f8703ec28b174b9e0408e4978ce9d8885b9f67e0e53fedde",
"crates": {
"addr2line 0.20.0": {
"name": "addr2line",
Expand Down Expand Up @@ -2501,6 +2501,36 @@
},
"license": "MIT OR Apache-2.0"
},
"dtoa 1.0.9": {
"name": "dtoa",
"version": "1.0.9",
"repository": {
"Http": {
"url": "https://crates.io/api/v1/crates/dtoa/1.0.9/download",
"sha256": "dcbb2bf8e87535c23f7a8a321e364ce21462d0ff10cb6407820e8e96dfff6653"
}
},
"targets": [
{
"Library": {
"crate_name": "dtoa",
"crate_root": "src/lib.rs",
"srcs": [
"**/*.rs"
]
}
}
],
"library_target_name": "dtoa",
"common_attrs": {
"compile_data_glob": [
"**"
],
"edition": "2018",
"version": "1.0.9"
},
"license": "MIT OR Apache-2.0"
},
"dummy_name_to_make_cargo_tooling_happy 0.0.0": {
"name": "dummy_name_to_make_cargo_tooling_happy",
"version": "0.0.0",
Expand Down Expand Up @@ -2611,6 +2641,10 @@
"id": "pin-project-lite 0.2.10",
"target": "pin_project_lite"
},
{
"id": "prometheus-client 0.21.2",
"target": "prometheus_client"
},
{
"id": "prost 0.11.9",
"target": "prost"
Expand Down Expand Up @@ -7768,6 +7802,133 @@
},
"license": "MIT OR Apache-2.0"
},
"prometheus-client 0.21.2": {
"name": "prometheus-client",
"version": "0.21.2",
"repository": {
"Http": {
"url": "https://crates.io/api/v1/crates/prometheus-client/0.21.2/download",
"sha256": "3c99afa9a01501019ac3a14d71d9f94050346f55ca471ce90c799a15c58f61e2"
}
},
"targets": [
{
"Library": {
"crate_name": "prometheus_client",
"crate_root": "src/lib.rs",
"srcs": [
"**/*.rs"
]
}
},
{
"BuildScript": {
"crate_name": "build_script_build",
"crate_root": "build.rs",
"srcs": [
"**/*.rs"
]
}
}
],
"library_target_name": "prometheus_client",
"common_attrs": {
"compile_data_glob": [
"**"
],
"crate_features": {
"common": [
"default"
],
"selects": {}
},
"deps": {
"common": [
{
"id": "dtoa 1.0.9",
"target": "dtoa"
},
{
"id": "itoa 1.0.9",
"target": "itoa"
},
{
"id": "parking_lot 0.12.1",
"target": "parking_lot"
},
{
"id": "prometheus-client 0.21.2",
"target": "build_script_build"
}
],
"selects": {}
},
"edition": "2021",
"proc_macro_deps": {
"common": [
{
"id": "prometheus-client-derive-encode 0.4.1",
"target": "prometheus_client_derive_encode"
}
],
"selects": {}
},
"version": "0.21.2"
},
"build_script_attrs": {
"data_glob": [
"**"
]
},
"license": "Apache-2.0 OR MIT"
},
"prometheus-client-derive-encode 0.4.1": {
"name": "prometheus-client-derive-encode",
"version": "0.4.1",
"repository": {
"Http": {
"url": "https://crates.io/api/v1/crates/prometheus-client-derive-encode/0.4.1/download",
"sha256": "72b6a5217beb0ad503ee7fa752d451c905113d70721b937126158f3106a48cc1"
}
},
"targets": [
{
"ProcMacro": {
"crate_name": "prometheus_client_derive_encode",
"crate_root": "src/lib.rs",
"srcs": [
"**/*.rs"
]
}
}
],
"library_target_name": "prometheus_client_derive_encode",
"common_attrs": {
"compile_data_glob": [
"**"
],
"deps": {
"common": [
{
"id": "proc-macro2 1.0.66",
"target": "proc_macro2"
},
{
"id": "quote 1.0.31",
"target": "quote"
},
{
"id": "syn 1.0.109",
"target": "syn"
}
],
"selects": {}
},
"edition": "2021",
"version": "0.4.1"
},
"license": "Apache-2.0 OR MIT"
},
"prost 0.11.9": {
"name": "prost",
"version": "0.11.9",
Expand Down
30 changes: 30 additions & 0 deletions Cargo.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ hashbrown = "0.14"
hyper = "0.14.27"
axum = "0.6.18"
tower = "0.4.13"
prometheus-client = "0.21.2"

[dev-dependencies]
stdext = "0.3.1"
Expand Down
1 change: 1 addition & 0 deletions cas/BUILD
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ rust_binary(
"@crate_index//:futures",
"@crate_index//:hyper",
"@crate_index//:json5",
"@crate_index//:prometheus-client",
"@crate_index//:axum",
"@crate_index//:tower",
"@crate_index//:tokio",
Expand Down
64 changes: 59 additions & 5 deletions cas/cas_main.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,10 @@ use axum::Router;
use clap::Parser;
use futures::future::{ok, select_all, BoxFuture, OptionFuture, TryFutureExt};
use hyper::service::make_service_fn;
use hyper::Server;
use hyper::{Body, Response, Server};
use prometheus_client::registry::Registry;
use runfiles::Runfiles;
use tokio::task::spawn_blocking;
use tonic::codec::CompressionEncoding;
use tonic::transport::Server as TonicServer;
use tower::util::ServiceExt;
Expand All @@ -41,6 +43,9 @@ use worker_api_server::WorkerApiServer;

const DEFAULT_CONFIG_FILE: &str = "<built-in example in config/examples/basic_cas.json>";

/// Note: This must be kept in sync with the documentation in `PrometheusConfig::path`.
const DEFAULT_PROMETHEUS_METRICS_PATH: &str = "/metrics";

/// Backend for bazel remote execution / cache API.
#[derive(Parser, Debug)]
#[clap(
Expand All @@ -57,6 +62,8 @@ struct Args {

#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
let mut root_metrics_registry = <Registry>::with_prefix("turbo_cache");

let args = Args::parse();
// Note: We cannot mutate args, so we create another variable for it here.
let mut config_file = args.config_file;
Expand Down Expand Up @@ -95,11 +102,13 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
};
set_open_file_limit(global_cfg.max_open_files);

let root_store_metrics = root_metrics_registry.sub_registry_with_prefix("stores");
let store_manager = Arc::new(StoreManager::new());
for (name, store_cfg) in cfg.stores {
let store_metrics = root_store_metrics.sub_registry_with_prefix(&name);
store_manager.add_store(
&name,
store_factory(&store_cfg, &store_manager)
store_factory(&store_cfg, &store_manager, store_metrics)
.await
.err_tip(|| format!("Failed to create store '{}'", name))?,
);
Expand Down Expand Up @@ -156,6 +165,8 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
}
}

// Lock our registry as immutable and clonable.
let root_metrics_registry = Arc::new(root_metrics_registry);
for server_cfg in cfg.servers {
let services = server_cfg.services.ok_or("'services' must be configured")?;

Expand Down Expand Up @@ -312,17 +323,60 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
.err_tip(|| "Could not create WorkerApi service")?,
);

let svc = Router::new()
let root_metrics_registry = root_metrics_registry.clone();

let mut svc = Router::new()
// This is the default service that executes if no other endpoint matches.
.fallback_service(tonic_services.into_service().map_err(|e| panic!("{e}")))
// This is a generic endpoint used to check if the server is up.
.route_service("/status", axum::routing::get(move || async move { "Ok".to_string() }));

let addr = server_cfg.listen_address.parse()?;
if let Some(prometheus_cfg) = services.prometheus {
fn error_to_response<E: std::error::Error>(e: E) -> hyper::Response<Body> {
hyper::Response::builder()
.status(500)
.body(format!("Error: {:?}", e).into())
.unwrap()
}
let path = if prometheus_cfg.path.is_empty() {
DEFAULT_PROMETHEUS_METRICS_PATH
} else {
&prometheus_cfg.path
};
svc = svc.route_service(
path,
axum::routing::get(move |_request: hyper::Request<hyper::Body>| async move {
// We spawn on a thread that can block to give more freedom to our metrics
// collection. This allows it to call functions like `tokio::block_in_place`
// if it needs to wait on a future.
spawn_blocking(move || {
let mut buf = String::new();
prometheus_client::encoding::text::encode(&mut buf, &root_metrics_registry)
.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))
.map(|_| {
let body = Body::from(buf);
Response::builder()
.header(
hyper::header::CONTENT_TYPE,
// Per spec we should probably use `application/openmetrics-text; version=1.0.0; charset=utf-8`
// https://github.com/OpenObservability/OpenMetrics/blob/1386544931307dff279688f332890c31b6c5de36/specification/OpenMetrics.md#overall-structure
// However, this makes debugging more difficult, so we use the old text/plain instead.
"text/plain; version=0.0.4; charset=utf-8",
)
.body(body)
.unwrap()
})
.unwrap_or_else(error_to_response)
})
.await
.unwrap_or_else(error_to_response)
}),
)
}

futures.push(Box::pin(
tokio::spawn(
Server::bind(&addr)
Server::bind(&server_cfg.listen_address.parse()?)
.serve(make_service_fn(move |_conn| ok::<_, Error>(svc.clone())))
.map_err(|e| make_err!(Code::Internal, "Failed running service : {:?}", e)),
)
Expand Down
3 changes: 3 additions & 0 deletions cas/grpc_service/BUILD
View file
Original file line number Diff line number Diff line change
Expand Up @@ -149,6 +149,7 @@ rust_test(
"//util:error",
"@crate_index//:maplit",
"@crate_index//:pretty_assertions",
"@crate_index//:prometheus-client",
"@crate_index//:tokio",
"@crate_index//:tonic",
],
Expand All @@ -168,6 +169,7 @@ rust_test(
"@crate_index//:bytes",
"@crate_index//:maplit",
"@crate_index//:pretty_assertions",
"@crate_index//:prometheus-client",
"@crate_index//:prost",
"@crate_index//:tokio",
"@crate_index//:tonic",
Expand All @@ -188,6 +190,7 @@ rust_test(
"@crate_index//:futures",
"@crate_index//:maplit",
"@crate_index//:pretty_assertions",
"@crate_index//:prometheus-client",
"@crate_index//:tokio",
"@crate_index//:tokio-stream",
"@crate_index//:tonic",
Expand Down
Loading

0 comments on commit 04a7772

Please sign in to comment.