diff --git a/Cargo.lock b/Cargo.lock index c2ad266..feba1d5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1055,7 +1055,7 @@ checksum = "adcf93614601c8129ddf72e2d5633df827ba6551541c6d8c59520a371475be1f" dependencies = [ "hermit-abi 0.3.1", "io-lifetimes", - "rustix", + "rustix 0.37.19", "windows-sys 0.48.0", ] @@ -1095,6 +1095,12 @@ version = "0.2.147" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" +[[package]] +name = "linux-raw-sys" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4" + [[package]] name = "linux-raw-sys" version = "0.3.8" @@ -1327,7 +1333,7 @@ dependencies = [ "libc", "redox_syscall 0.3.5", "smallvec", - "windows-targets", + "windows-targets 0.48.0", ] [[package]] @@ -1407,6 +1413,42 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "procfs" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1de8dacb0873f77e6aefc6d71e044761fcc68060290f5b1089fcdf84626bb69" +dependencies = [ + "bitflags 1.3.2", + "byteorder", + "hex", + "lazy_static", + "rustix 0.36.14", +] + +[[package]] +name = "prometheus" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "449811d15fbdf5ceb5c1144416066429cf82316e2ec8ce0c1f6f8a02e7bbcf8c" +dependencies = [ + "cfg-if", + "fnv", + "lazy_static", + "libc", + "memchr", + "parking_lot", + "procfs", + "protobuf", + "thiserror", +] + +[[package]] +name = "protobuf" +version = "2.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94" + [[package]] name = "pwd" version = "1.4.0" @@ -1564,15 +1606,29 @@ dependencies = [ [[package]] name = "rustix" -version = "0.37.20" +version = "0.36.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14e4d67015953998ad0eb82887a0eb0129e18a7e2f3b7b0f6c422fddcd503d62" +dependencies = [ + "bitflags 1.3.2", + "errno", + "io-lifetimes", + "libc", + "linux-raw-sys 0.1.4", + "windows-sys 0.45.0", +] + +[[package]] +name = "rustix" +version = "0.37.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b96e891d04aa506a6d1f318d2771bcb1c7dfda84e126660ace067c9b474bb2c0" +checksum = "acf8729d8542766f1b2cf77eb034d52f40d375bb8b615d0b147089946e16613d" dependencies = [ "bitflags 1.3.2", "errno", "io-lifetimes", "libc", - "linux-raw-sys", + "linux-raw-sys 0.3.8", "windows-sys 0.48.0", ] @@ -1637,11 +1693,13 @@ dependencies = [ "expanduser", "http", "hyper", + "lazy_static", "maligned", "mime", "ndarray", "ndarray-stats", "num-traits", + "prometheus", "regex", "serde", "serde_json", @@ -2396,13 +2454,37 @@ dependencies = [ "windows_x86_64_msvc 0.42.2", ] +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets 0.42.2", +] + [[package]] name = "windows-sys" version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" dependencies = [ - "windows-targets", + "windows-targets 0.48.0", +] + +[[package]] +name = "windows-targets" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 5b783e4..059d11f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,11 +20,13 @@ clap = { version = "4.2", features = ["derive", "env"] } expanduser = "1.2.2" http = "*" hyper = { version = "0.14", features = ["full"] } +lazy_static = "1.4" maligned = "0.2.1" mime = "0.3" ndarray = "0.15" ndarray-stats = "0.5" num-traits = "0.2.15" +prometheus = { version = "0.13", features = ["process"] } serde = { version = "1.0", features = ["derive"] } serde_json = "*" strum_macros = "0.24" diff --git a/scripts/docker-compose.yml b/scripts/docker-compose.yml new file mode 100644 index 0000000..5413e9c --- /dev/null +++ b/scripts/docker-compose.yml @@ -0,0 +1,22 @@ +# NOTE: This should only be used for explicitly testing the prometheus metrics +# since it builds the Rust project from scratch within the container each time +# so is very slow and inefficient for regular development work. +services: + active-storage-proxy: + build: ../. + ports: + - "8080:8080" + minio: + image: minio/minio + command: ["server", "data", "--console-address", ":9001"] + ports: + - "9000:9000" + - "9001:9001" + prometheus: + image: prom/prometheus + volumes: + - type: bind + source: ./prometheus.yml + target: /etc/prometheus/prometheus.yml + ports: + - "9090:9090" \ No newline at end of file diff --git a/scripts/prometheus.yml b/scripts/prometheus.yml new file mode 100644 index 0000000..2b23068 --- /dev/null +++ b/scripts/prometheus.yml @@ -0,0 +1,7 @@ +global: + scrape_interval: 5s +scrape_configs: + - job_name: active-storage-proxy + static_configs: + - targets: + - "active-storage-proxy:8080" \ No newline at end of file diff --git a/src/app.rs b/src/app.rs index d8caf9e..8818e72 100644 --- a/src/app.rs +++ b/src/app.rs @@ -1,12 +1,14 @@ //! Active Storage server API use crate::error::ActiveStorageError; +use crate::metrics::{metrics_handler, track_metrics}; use crate::models; use crate::operation; use crate::operations; use crate::s3_client; use crate::validated_json::ValidatedJson; +use axum::middleware; use axum::{ body::{Body, Bytes}, extract::Path, @@ -86,7 +88,9 @@ fn router() -> Router { Router::new() .route("/.well-known/s3-active-storage-schema", get(schema)) + .route("/metrics", get(metrics_handler)) .nest("/v1", v1()) + .route_layer(middleware::from_fn(track_metrics)) } /// S3 Active Storage Server Service type alias diff --git a/src/lib.rs b/src/lib.rs index 27bdb74..391b4f4 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -26,6 +26,7 @@ pub mod app; pub mod array; pub mod cli; pub mod error; +pub mod metrics; pub mod models; pub mod operation; pub mod operations; diff --git a/src/main.rs b/src/main.rs index a8edc9c..07b8a41 100644 --- a/src/main.rs +++ b/src/main.rs @@ -2,6 +2,7 @@ use s3_active_storage::app; use s3_active_storage::cli; +use s3_active_storage::metrics; use s3_active_storage::server; use s3_active_storage::tracing; @@ -10,6 +11,7 @@ use s3_active_storage::tracing; async fn main() { let args = cli::parse(); tracing::init_tracing(); + metrics::register_metrics(); let service = app::service(); server::serve(&args, service).await; } diff --git a/src/metrics.rs b/src/metrics.rs new file mode 100644 index 0000000..20d4d38 --- /dev/null +++ b/src/metrics.rs @@ -0,0 +1,83 @@ +use std::time::Instant; + +use axum::{http::Request, middleware::Next, response::IntoResponse}; +use lazy_static::lazy_static; +use prometheus::{self, Encoder, HistogramOpts, HistogramVec, IntCounterVec, Opts}; + +lazy_static! { + // Simple request counter + pub static ref INCOMING_REQUESTS: IntCounterVec = IntCounterVec::new( + Opts::new("incoming_requests", "The number of HTTP requests received"), + &["http_method", "path"] + ).expect("Prometheus metric initialization failed"); + // Request counter by status code + pub static ref RESPONSE_CODE_COLLECTOR: IntCounterVec = IntCounterVec::new( + Opts::new("outgoing_response", "The number of responses sent"), + &["status_code", "http_method", "path"] + ).expect("Prometheus metric initialization failed"); + // Response histogram by response time + pub static ref RESPONSE_TIME_COLLECTOR: HistogramVec = HistogramVec::new( + HistogramOpts{ + common_opts: Opts::new("response_time", "The time taken to respond to each request"), + buckets: prometheus::DEFAULT_BUCKETS.to_vec(), // Change buckets here if desired + }, + &["status_code", "http_method", "path"], + ).expect("Prometheus metric initialization failed"); +} + +/// Registers various prometheus metrics with the global registry +pub fn register_metrics() { + let registry = prometheus::default_registry(); + registry + .register(Box::new(INCOMING_REQUESTS.clone())) + .expect("registering prometheus metrics during initialization failed"); + registry + .register(Box::new(RESPONSE_CODE_COLLECTOR.clone())) + .expect("registering prometheus metrics during initialization failed"); + registry + .register(Box::new(RESPONSE_TIME_COLLECTOR.clone())) + .expect("registering prometheus metrics during initialization failed"); +} + +/// Returns currently gathered prometheus metrics +pub async fn metrics_handler() -> String { + let encoder = prometheus::TextEncoder::new(); + let mut buffer = Vec::new(); + + encoder + .encode(&prometheus::gather(), &mut buffer) + .expect("could not encode gathered metrics into temporary buffer"); + + String::from_utf8(buffer).expect("could not convert metrics buffer into string") +} + +pub async fn track_metrics(request: Request, next: Next) -> impl IntoResponse { + // Extract some useful quantities + let timer = Instant::now(); + let http_method = &request.method().to_string().to_ascii_uppercase(); + let request_path = request.uri().path().to_string(); + + // Increment request counter + INCOMING_REQUESTS + .with_label_values(&[http_method, &request_path]) + .inc(); + + // Pass request onto next layer + let response = next.run(request).await; + let status_code = response.status(); + // Due to 'concentric shell model' for axum layers, + // latency is time taken to traverse all inner + // layers (including primary reduction operation) + // and then back up the layer stack. + let latency = timer.elapsed().as_secs_f64(); + + // Record response metrics + RESPONSE_CODE_COLLECTOR + .with_label_values(&[status_code.as_str(), http_method, &request_path]) + .inc(); + RESPONSE_TIME_COLLECTOR + .with_label_values(&[status_code.as_str(), http_method, &request_path]) + .observe(latency); + + response +}