diff --git a/Cargo.lock b/Cargo.lock index ad212185..1aa064c7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1835,7 +1835,6 @@ dependencies = [ "blsful", "cb-common", "cb-metrics", - "client-ip", "eyre", "futures", "headers", @@ -1994,16 +1993,6 @@ version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b94f61472cee1439c0b966b47e3aca9ae07e45d070759512cd390ea2bebc6675" -[[package]] -name = "client-ip" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31211fc26899744f5b22521fdc971e5f3875991d8880537537470685a0e9552d" -dependencies = [ - "forwarded-header-value", - "http 1.3.1", -] - [[package]] name = "cmake" version = "0.1.54" @@ -3248,16 +3237,6 @@ dependencies = [ "percent-encoding", ] -[[package]] -name = "forwarded-header-value" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8835f84f38484cc86f110a805655697908257fb9a7af005234060891557198e9" -dependencies = [ - "nonempty", - "thiserror 1.0.69", -] - [[package]] name = "fs-err" version = "3.1.2" @@ -4562,12 +4541,6 @@ dependencies = [ "minimal-lexical", ] -[[package]] -name = "nonempty" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9e591e719385e6ebaeb5ce5d3887f7d5676fceca6411d1925ccc95745f3d6f7" - [[package]] name = "nu-ansi-term" version = "0.50.1" diff --git a/Cargo.toml b/Cargo.toml index 64bf5d4d..f27b7d4e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,7 +35,6 @@ cb-pbs = { path = "crates/pbs" } cb-signer = { path = "crates/signer" } cipher = "0.4" clap = { version = "4.5.4", features = ["derive", "env"] } -client-ip = { version = "0.1.1", features = [ "forwarded-header" ] } color-eyre = "0.6.3" const_format = "0.2.34" ctr = "0.9.2" diff --git a/config.example.toml b/config.example.toml index f5e42e1c..94824984 100644 --- a/config.example.toml +++ b/config.example.toml @@ -170,6 +170,14 @@ jwt_auth_fail_limit = 3 # OPTIONAL, DEFAULT: 300 jwt_auth_fail_timeout_seconds = 300 +# HTTP header to use to determine the real client IP, if the Signer is behind a proxy (e.g. nginx) +# OPTIONAL. If missing, the client IP will be taken directly from the TCP connection. +# [signer.reverse_proxy] +# Unique: HTTP header name to use to determine the real client IP. Expected to appear only once in the request. Requests with multiple values of this header will be rejected. +# unique = "X-Real-IP" +# Rightmost: HTTP header name to use to determine the real client IP from a comma-separated list of IPs. Rightmost IP is the client IP. If the header appears multiple times, the last value will be used. +# rightmost = "X-Forwarded-For" + # [signer.tls_mode] # How to use TLS for the Signer's HTTP server; two modes are supported: # - type = "insecure": disable TLS, so the server runs in HTTP mode (not recommended for production). diff --git a/crates/common/src/commit/client.rs b/crates/common/src/commit/client.rs index b614ab0f..98d8c26d 100644 --- a/crates/common/src/commit/client.rs +++ b/crates/common/src/commit/client.rs @@ -2,10 +2,7 @@ use std::path::PathBuf; use alloy::primitives::Address; use eyre::WrapErr; -use reqwest::{ - Certificate, - header::{AUTHORIZATION, HeaderMap, HeaderValue}, -}; +use reqwest::Certificate; use serde::{Deserialize, Serialize}; use url::Url; @@ -26,9 +23,7 @@ use crate::{ }, response::{BlsSignResponse, EcdsaSignResponse}, }, - constants::SIGNER_JWT_EXPIRATION, - signer::EcdsaSignature, - types::{BlsPublicKey, BlsSignature, Jwt, ModuleId}, + types::{BlsPublicKey, Jwt, ModuleId}, utils::create_jwt, }; diff --git a/crates/common/src/config/signer.rs b/crates/common/src/config/signer.rs index 7cbfb2e7..98608413 100644 --- a/crates/common/src/config/signer.rs +++ b/crates/common/src/config/signer.rs @@ -69,6 +69,16 @@ pub enum TlsMode { Certificate(PathBuf), } +/// Reverse proxy setup, used to extract real client's IP +#[derive(Debug, Serialize, Deserialize, Clone, Default)] +#[serde(rename_all = "snake_case")] +pub enum ReverseProxyHeaderSetup { + #[default] + None, + Unique(String), + Rightmost(String), +} + #[derive(Debug, Serialize, Deserialize, Clone)] #[serde(rename_all = "snake_case")] pub struct SignerConfig { @@ -99,6 +109,10 @@ pub struct SignerConfig { #[serde(default = "default_tls_mode")] pub tls_mode: TlsMode, + /// Reverse proxy setup to extract real client's IP + #[serde(default)] + pub reverse_proxy: ReverseProxyHeaderSetup, + /// Inner type-specific configuration #[serde(flatten)] pub inner: SignerType, @@ -194,6 +208,7 @@ pub struct StartSignerConfig { pub jwt_auth_fail_timeout_seconds: u32, pub dirk: Option, pub tls_certificates: Option<(Vec, Vec)>, + pub reverse_proxy: ReverseProxyHeaderSetup, } impl StartSignerConfig { @@ -247,6 +262,8 @@ impl StartSignerConfig { } }; + let reverse_proxy = signer_config.reverse_proxy; + match signer_config.inner { SignerType::Local { loader, store, .. } => Ok(StartSignerConfig { chain: config.chain, @@ -259,6 +276,7 @@ impl StartSignerConfig { store, dirk: None, tls_certificates, + reverse_proxy, }), SignerType::Dirk { @@ -305,6 +323,7 @@ impl StartSignerConfig { }, }), tls_certificates, + reverse_proxy, }) } diff --git a/crates/signer/Cargo.toml b/crates/signer/Cargo.toml index 1a688e1b..7c6e63fa 100644 --- a/crates/signer/Cargo.toml +++ b/crates/signer/Cargo.toml @@ -14,7 +14,6 @@ bimap.workspace = true blsful.workspace = true cb-common.workspace = true cb-metrics.workspace = true -client-ip.workspace = true eyre.workspace = true futures.workspace = true headers.workspace = true diff --git a/crates/signer/src/lib.rs b/crates/signer/src/lib.rs index 4b5e1451..b4b9ecc4 100644 --- a/crates/signer/src/lib.rs +++ b/crates/signer/src/lib.rs @@ -4,3 +4,4 @@ pub mod manager; mod metrics; mod proto; pub mod service; +mod utils; diff --git a/crates/signer/src/service.rs b/crates/signer/src/service.rs index 1116e764..219a4ae5 100644 --- a/crates/signer/src/service.rs +++ b/crates/signer/src/service.rs @@ -30,13 +30,12 @@ use cb_common::{ }, response::{BlsSignResponse, EcdsaSignResponse}, }, - config::{ModuleSigningConfig, StartSignerConfig}, + config::{ModuleSigningConfig, ReverseProxyHeaderSetup, StartSignerConfig}, constants::{COMMIT_BOOST_COMMIT, COMMIT_BOOST_VERSION}, types::{BlsPublicKey, Chain, Jwt, ModuleId, SignatureRequestInfo}, utils::{decode_jwt, validate_admin_jwt, validate_jwt}, }; use cb_metrics::provider::MetricsProvider; -use client_ip::*; use eyre::Context; use headers::{Authorization, authorization::Bearer}; use parking_lot::RwLock as ParkingRwLock; @@ -49,6 +48,7 @@ use crate::{ error::SignerModuleError, manager::{SigningManager, dirk::DirkManager, local::LocalSigningManager}, metrics::{SIGNER_METRICS_REGISTRY, SIGNER_STATUS, uri_to_tag}, + utils::get_true_ip, }; pub const REQUEST_MAX_BODY_LENGTH: usize = 1024 * 1024; // 1 MB @@ -83,6 +83,9 @@ struct SigningState { // JWT auth failure settings jwt_auth_fail_limit: u32, jwt_auth_fail_timeout: Duration, + + /// Header to extract the trusted client IP from + reverse_proxy: ReverseProxyHeaderSetup, } impl SigningService { @@ -102,6 +105,7 @@ impl SigningService { jwt_auth_failures: Arc::new(ParkingRwLock::new(HashMap::new())), jwt_auth_fail_limit: config.jwt_auth_fail_limit, jwt_auth_fail_timeout: Duration::from_secs(config.jwt_auth_fail_timeout_seconds as u64), + reverse_proxy: config.reverse_proxy, }; // Get the signer counts @@ -122,6 +126,7 @@ impl SigningService { loaded_proxies, jwt_auth_fail_limit =? state.jwt_auth_fail_limit, jwt_auth_fail_timeout =? state.jwt_auth_fail_timeout, + reverse_proxy =? state.reverse_proxy, "Starting signing service" ); @@ -224,38 +229,6 @@ fn mark_jwt_failure(state: &SigningState, client_ip: IpAddr) { failure_info.last_failure = Instant::now(); } -/// Get the true client IP from the request headers or fallback to the socket -/// address -fn get_true_ip(req_headers: &HeaderMap, addr: &SocketAddr) -> eyre::Result { - let ip_extractors = [ - cf_connecting_ip, - cloudfront_viewer_address, - fly_client_ip, - rightmost_forwarded, - rightmost_x_forwarded_for, - true_client_ip, - x_real_ip, - ]; - - // Run each extractor in order and return the first valid IP found - for extractor in ip_extractors { - match extractor(req_headers) { - Ok(true_ip) => { - return Ok(true_ip); - } - Err(e) => { - match e { - Error::AbsentHeader { .. } => continue, // Missing headers are fine - _ => return Err(eyre::eyre!(e.to_string())), // Report anything else - } - } - } - } - - // Fallback to the socket IP - Ok(addr.ip()) -} - /// Authentication middleware layer async fn jwt_auth( State(state): State, @@ -266,7 +239,7 @@ async fn jwt_auth( next: Next, ) -> Result { // Check if the request needs to be rate limited - let client_ip = get_true_ip(&req_headers, &addr).map_err(|e| { + let client_ip = get_true_ip(&req_headers, &addr, &state.reverse_proxy).map_err(|e| { error!("Failed to get client IP: {e}"); SignerModuleError::RequestError("failed to get client IP".to_string()) })?; @@ -376,7 +349,7 @@ async fn admin_auth( next: Next, ) -> Result { // Check if the request needs to be rate limited - let client_ip = get_true_ip(&req_headers, &addr).map_err(|e| { + let client_ip = get_true_ip(&req_headers, &addr, &state.reverse_proxy).map_err(|e| { error!("Failed to get client IP: {e}"); SignerModuleError::RequestError("failed to get client IP".to_string()) })?; diff --git a/crates/signer/src/utils.rs b/crates/signer/src/utils.rs new file mode 100644 index 00000000..a6d313ef --- /dev/null +++ b/crates/signer/src/utils.rs @@ -0,0 +1,64 @@ +use std::net::{IpAddr, SocketAddr}; + +use axum::http::HeaderMap; +use cb_common::config::ReverseProxyHeaderSetup; + +#[derive(Debug, thiserror::Error)] +pub enum IpError { + #[error("header `{0}` is not present")] + NotPresent(String), + #[error("header value has invalid characters")] + HasInvalidCharacters, + #[error("header value is not a valid IP address")] + InvalidValue, + #[error("header `{0}` appears multiple times but expected to be unique")] + NotUnique(String), +} + +/// Get the true client IP from the request headers or fallback to the socket +/// address +pub fn get_true_ip( + headers: &HeaderMap, + addr: &SocketAddr, + reverse_proxy: &ReverseProxyHeaderSetup, +) -> Result { + match reverse_proxy { + ReverseProxyHeaderSetup::None => Ok(addr.ip()), + ReverseProxyHeaderSetup::Unique(header) => get_ip_from_unique_header(headers, header), + ReverseProxyHeaderSetup::Rightmost(header) => get_ip_from_rightmost_value(headers, header), + } +} + +fn get_ip_from_unique_header(headers: &HeaderMap, header_name: &str) -> Result { + let mut values = headers.get_all(header_name).iter(); + + let first_value = values.next().ok_or(IpError::NotPresent(header_name.to_string()))?; + let ip = first_value + .to_str() + .map_err(|_| IpError::HasInvalidCharacters)? + .parse::() + .map_err(|_| IpError::InvalidValue)?; + + if values.next().is_some() { + return Err(IpError::NotUnique(header_name.to_string())); + } + + Ok(ip) +} + +fn get_ip_from_rightmost_value(headers: &HeaderMap, header_name: &str) -> Result { + let last_value = headers + .get_all(header_name) + .iter() + .next_back() + .ok_or(IpError::NotPresent(header_name.to_string()))? + .to_str() + .map_err(|_| IpError::HasInvalidCharacters)?; + + last_value + .rsplit_once(",") + .map(|(_, rightmost)| rightmost) + .unwrap_or(last_value) + .parse::() + .map_err(|_| IpError::InvalidValue) +} diff --git a/docs/docs/get_started/configuration.md b/docs/docs/get_started/configuration.md index acff09e7..abdb9e20 100644 --- a/docs/docs/get_started/configuration.md +++ b/docs/docs/get_started/configuration.md @@ -388,6 +388,33 @@ path = "path/to/your/cert/folder" Where `path` is the aforementioned folder. It defaults to `./certs` but can be replaced with whichever directory your certificate and private key file reside in, as long as they're readable by the Signer service (or its Docker container, if using Docker). +### Rate limit + +The Signer service implements a rate limit system of 3 failed authentications every 5 minutes. These values can be modified in the config file: + +```toml +[signer] +... +jwt_auth_fail_limit = 3 # The amount of failed requests allowed +jwt_auth_fail_timeout_seconds = 300 # The time window in seconds +``` + +The rate limit is applied to the IP address of the client making the request. By default, the IP is extracted directly from the TCP connection. If you're running the Signer service behind a reverse proxy (e.g. Nginx), you can configure it to extract the IP from a custom HTTP header instead. There're two options: + +- `unique`: The name of the HTTP header that contains the IP. This header is expected to appear only once in the request. This is common when using `X-Real-IP`, `True-Client-IP`, etc. If a request is received that has multiple values for this header, it will be considered invalid and rejected. +- `rightmost`: The name of the HTTP header that contains a comma-separated list of IPs. The rightmost IP in the list is used. If the header appears multiple times, the last occurrence is used. This is common when using `X-Forwarded-For`. + +Examples: + +```toml +[signer.reverse_proxy] +unique = "X-Real-IP" +``` + +```toml +[signer.reverse_proxy] +rightmost = "X-Forwarded-For" +``` ## Custom module diff --git a/tests/src/utils.rs b/tests/src/utils.rs index 17d54025..330f1e64 100644 --- a/tests/src/utils.rs +++ b/tests/src/utils.rs @@ -9,9 +9,10 @@ use alloy::primitives::{B256, U256}; use cb_common::{ config::{ CommitBoostConfig, LogsSettings, ModuleKind, ModuleSigningConfig, PbsConfig, - PbsModuleConfig, RelayConfig, SIGNER_IMAGE_DEFAULT, SIGNER_JWT_AUTH_FAIL_LIMIT_DEFAULT, - SIGNER_JWT_AUTH_FAIL_TIMEOUT_SECONDS_DEFAULT, SIGNER_PORT_DEFAULT, SignerConfig, - SignerType, StartSignerConfig, StaticModuleConfig, StaticPbsConfig, TlsMode, + PbsModuleConfig, RelayConfig, ReverseProxyHeaderSetup, SIGNER_IMAGE_DEFAULT, + SIGNER_JWT_AUTH_FAIL_LIMIT_DEFAULT, SIGNER_JWT_AUTH_FAIL_TIMEOUT_SECONDS_DEFAULT, + SIGNER_PORT_DEFAULT, SignerConfig, SignerType, StartSignerConfig, StaticModuleConfig, + StaticPbsConfig, TlsMode, }, pbs::{RelayClient, RelayEntry}, signer::SignerLoader, @@ -130,6 +131,7 @@ pub fn get_signer_config(loader: SignerLoader, tls: bool) -> SignerConfig { jwt_auth_fail_timeout_seconds: SIGNER_JWT_AUTH_FAIL_TIMEOUT_SECONDS_DEFAULT, inner: SignerType::Local { loader, store: None }, tls_mode: if tls { TlsMode::Certificate(PathBuf::new()) } else { TlsMode::Insecure }, + reverse_proxy: ReverseProxyHeaderSetup::None, } } @@ -165,6 +167,7 @@ pub fn get_start_signer_config( jwt_auth_fail_timeout_seconds: signer_config.jwt_auth_fail_timeout_seconds, dirk: None, tls_certificates, + reverse_proxy: ReverseProxyHeaderSetup::None, }, _ => panic!("Only local signers are supported in tests"), }