Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions architecture/gateway.md
Original file line number Diff line number Diff line change
Expand Up @@ -60,8 +60,11 @@ checks the returned pod binding against the live pod UID, and verifies the pod's
controlling `Sandbox` ownerReference against the live Sandbox CR UID and
sandbox-id label before minting the gateway JWT. Supervisors renew gateway JWTs
in memory before expiry only while the sandbox record still exists. Older tokens
are not server-revoked; deployments bound replay exposure with short
`gateway_jwt.ttl_secs` lifetimes.
are not server-revoked; shared deployments bound replay exposure with short
`gateway_jwt.ttl_secs` lifetimes. The config default is
`gateway_jwt.ttl_secs = 0` for local single-player Docker, Podman, and VM
gateways; those tokens carry `exp = 0` and do not expire. Kubernetes and other
shared deployments should set a positive TTL.

Gateway JWT signing-key rotation is currently an offline operator action. The
runtime loads one active signing key and one matching public verification key
Expand Down
19 changes: 16 additions & 3 deletions crates/openshell-core/src/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -504,7 +504,8 @@ pub struct GatewayJwtConfig {
/// hostname-or-`openshell` placeholder if unset.
#[serde(default = "default_gateway_id")]
pub gateway_id: String,
/// Token lifetime in seconds. Defaults to 1 hour.
/// Token lifetime in seconds. A value of 0 disables expiration and is
/// intended only for local single-player deployments.
#[serde(default = "default_sandbox_token_ttl_secs")]
pub ttl_secs: u64,
}
Expand All @@ -514,7 +515,7 @@ fn default_gateway_id() -> String {
}

const fn default_sandbox_token_ttl_secs() -> u64 {
3_600
0
}

fn default_roles_claim() -> String {
Expand Down Expand Up @@ -726,7 +727,7 @@ mod tests {
#[cfg(unix)]
use super::is_reachable_unix_socket;
use super::{
ComputeDriverKind, Config, DEFAULT_SERVICE_ROUTING_DOMAIN, detect_driver,
ComputeDriverKind, Config, DEFAULT_SERVICE_ROUTING_DOMAIN, GatewayJwtConfig, detect_driver,
docker_host_unix_socket_path, is_unix_socket, podman_socket_candidates_from_env,
podman_socket_responds,
};
Expand Down Expand Up @@ -781,6 +782,18 @@ mod tests {
assert!(!cfg.auth.allow_unauthenticated_users);
}

#[test]
fn gateway_jwt_ttl_defaults_to_non_expiring() {
let cfg: GatewayJwtConfig = serde_json::from_value(serde_json::json!({
"signing_key_path": "/tmp/signing.pem",
"public_key_path": "/tmp/public.pem",
"kid_path": "/tmp/kid"
}))
.expect("gateway JWT config should deserialize with default ttl");

assert_eq!(cfg.ttl_secs, 0);
}

#[test]
fn service_routing_allows_loopback_plaintext_http_by_default() {
let cfg = Config::new(None);
Expand Down
31 changes: 25 additions & 6 deletions crates/openshell-sandbox/src/grpc_client.rs
Original file line number Diff line number Diff line change
Expand Up @@ -389,7 +389,7 @@ async fn refresh_token_loop(
/// Compute the next refresh delay: 80 % of the time remaining until the
/// current token's `exp`, plus up to 10 % jitter, with a small lower bound
/// for already-expired tokens and capped at 12 h. If the token can't be parsed
/// (legacy/non-JWT bearer)
/// (legacy/non-JWT bearer) or carries the `exp = 0` non-expiring sentinel,
/// default to 6 h.
fn compute_refresh_delay(slot: &TokenSlot) -> Duration {
let token = slot
Expand All @@ -404,11 +404,16 @@ fn compute_refresh_delay(slot: &TokenSlot) -> Duration {
.map_or(0, |d| d.as_millis()),
)
.unwrap_or(i64::MAX);
let remaining_ms = parse_jwt_exp_ms(bearer).map_or(21_600_000, |exp| exp - now_ms); // 6 h fallback
let mut delay_ms = if remaining_ms <= 0 {
1_000
} else {
(remaining_ms * 8 / 10).clamp(1_000, 43_200_000)
let mut delay_ms = match parse_jwt_exp_ms(bearer) {
Some(0) | None => 21_600_000,
Some(exp) => {
let remaining_ms = exp - now_ms;
if remaining_ms <= 0 {
1_000
} else {
(remaining_ms * 8 / 10).clamp(1_000, 43_200_000)
}
}
};
// Up to 10 % jitter, derived deterministically from token bytes so
// unit tests are reproducible without injecting an RNG.
Expand Down Expand Up @@ -494,6 +499,20 @@ mod auth_tests {
assert!((1..60).contains(&delay.as_secs()));
}

#[test]
fn compute_refresh_delay_treats_exp_zero_as_non_expiring() {
use base64::Engine as _;
let payload = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(r#"{"exp":0}"#);
let token = format!("h.{payload}.s");
let bearer = AsciiMetadataValue::try_from(format!("Bearer {token}")).unwrap();
let slot: TokenSlot = Arc::new(RwLock::new(bearer));
let delay = compute_refresh_delay(&slot);
assert!(
(6 * 60 * 60..=7 * 60 * 60).contains(&delay.as_secs()),
"non-expiring tokens should use the fallback refresh delay, got {delay:?}"
);
}

#[test]
fn compute_refresh_delay_supports_short_token_ttl() {
use base64::Engine as _;
Expand Down
53 changes: 51 additions & 2 deletions crates/openshell-server/src/auth/sandbox_jwt.rs
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ use tracing::{debug, warn};
/// reuse the same subject namespace without breaking handler equality
/// checks.
const SPIFFE_SUBJECT_PREFIX: &str = "spiffe://openshell/sandbox/";
const SANDBOX_JWT_EXP_LEEWAY_SECS: i64 = 60;

/// JWT claim set serialized in every gateway-minted sandbox token.
#[derive(Debug, Serialize, Deserialize)]
Expand Down Expand Up @@ -100,7 +101,11 @@ impl SandboxJwtIssuer {
#[allow(clippy::result_large_err)] // `tonic::Status` is the natural error here
pub fn mint(&self, sandbox_id: &str) -> Result<MintedToken, Status> {
let now = now_secs();
let exp = now + i64::try_from(self.ttl.as_secs()).unwrap_or(3_600);
let exp = if self.ttl.is_zero() {
0
} else {
now.saturating_add(i64::try_from(self.ttl.as_secs()).unwrap_or(3_600))
};
let claims = SandboxJwtClaims {
sub: format!("{SPIFFE_SUBJECT_PREFIX}{sandbox_id}"),
iss: self.issuer.clone(),
Expand Down Expand Up @@ -178,6 +183,7 @@ impl SandboxJwtAuthenticator {
validation.set_issuer(&[&self.issuer]);
validation.set_audience(&[&self.audience]);
validation.set_required_spec_claims(&["iss", "aud", "exp", "sub"]);
validation.validate_exp = false;

let data =
decode::<SandboxJwtClaims>(token, &self.decoding_key, &validation).map_err(|e| {
Expand All @@ -186,6 +192,7 @@ impl SandboxJwtAuthenticator {
})?;

let claims = data.claims;
validate_exp(claims.exp)?;
Ok(Some(Principal::Sandbox(SandboxPrincipal {
sandbox_id: claims.sandbox_id,
source: SandboxIdentitySource::BootstrapJwt { issuer: claims.iss },
Expand All @@ -212,6 +219,20 @@ impl Authenticator for SandboxJwtAuthenticator {
}
}

#[allow(clippy::result_large_err)]
fn validate_exp(exp: i64) -> Result<(), Status> {
if exp == 0 {
return Ok(());
}

if exp < now_secs().saturating_sub(SANDBOX_JWT_EXP_LEEWAY_SECS) {
debug!("sandbox JWT expired");
return Err(Status::unauthenticated("invalid token: ExpiredSignature"));
}

Ok(())
}

fn now_secs() -> i64 {
i64::try_from(
SystemTime::now()
Expand All @@ -236,12 +257,16 @@ mod tests {
}

fn pair() -> (SandboxJwtIssuer, SandboxJwtAuthenticator) {
pair_with_ttl(Duration::from_secs(3600))
}

fn pair_with_ttl(ttl: Duration) -> (SandboxJwtIssuer, SandboxJwtAuthenticator) {
let mat = generate_jwt_key().expect("jwt key");
let issuer = SandboxJwtIssuer::from_pem(
mat.signing_key_pem.as_bytes(),
mat.kid.clone(),
"test-gateway",
Duration::from_secs(3600),
ttl,
)
.unwrap();
let auth = SandboxJwtAuthenticator::from_pem(
Expand Down Expand Up @@ -276,6 +301,30 @@ mod tests {
}
}

#[tokio::test]
async fn ttl_zero_mints_non_expiring_token() {
let (issuer, auth) = pair_with_ttl(Duration::ZERO);
let minted = issuer.mint("sandbox-never").unwrap();
assert_eq!(minted.expires_at_ms, 0);

let principal = auth
.authenticate(&header_map_with_bearer(&minted.token), "/anything")
.await
.unwrap()
.expect("exp=0 token should authenticate");
assert!(matches!(principal, Principal::Sandbox(_)));

let mut validation = Validation::new(Algorithm::EdDSA);
validation.algorithms = vec![Algorithm::EdDSA];
validation.set_issuer(&["openshell-gateway:test-gateway"]);
validation.set_audience(&["openshell-gateway:test-gateway"]);
validation.set_required_spec_claims(&["iss", "aud", "exp", "sub"]);
validation.validate_exp = false;
let decoded = decode::<SandboxJwtClaims>(&minted.token, &auth.decoding_key, &validation)
.expect("token should decode");
assert_eq!(decoded.claims.exp, 0);
}

#[tokio::test]
async fn token_signed_by_other_key_is_rejected() {
let (_, auth_a) = pair();
Expand Down
47 changes: 43 additions & 4 deletions crates/openshell-server/src/cli.rs
Original file line number Diff line number Diff line change
Expand Up @@ -618,6 +618,13 @@ fn effective_single_driver(args: &RunArgs) -> Option<ComputeDriverKind> {
}
}

fn is_singleplayer_driver(args: &RunArgs) -> bool {
matches!(
effective_single_driver(args),
Some(ComputeDriverKind::Docker | ComputeDriverKind::Podman | ComputeDriverKind::Vm)
)
}

fn resolve_mtls_auth_enabled(
args: &RunArgs,
matches: &ArgMatches,
Expand All @@ -634,10 +641,7 @@ fn resolve_mtls_auth_enabled(
return false;
}

matches!(
effective_single_driver(args),
Some(ComputeDriverKind::Docker | ComputeDriverKind::Podman | ComputeDriverKind::Vm)
)
is_singleplayer_driver(args)
}

/// Build [`VmComputeConfig`] from the `[openshell.drivers.vm]` table
Expand Down Expand Up @@ -1376,6 +1380,41 @@ ssh_session_ttl_secs = 1234
assert_eq!(file.openshell.gateway.ssh_session_ttl_secs, Some(1234));
}

#[test]
fn singleplayer_driver_matches_only_one_local_driver() {
for driver in ["docker", "podman", "vm"] {
let (args, _) = parse_with_args(&[
"openshell-gateway",
"--db-url",
"sqlite::memory:",
"--drivers",
driver,
]);
assert!(
super::is_singleplayer_driver(&args),
"{driver} should be singleplayer"
);
}

let (k8s, _) = parse_with_args(&[
"openshell-gateway",
"--db-url",
"sqlite::memory:",
"--drivers",
"kubernetes",
]);
assert!(!super::is_singleplayer_driver(&k8s));

let (multi, _) = parse_with_args(&[
"openshell-gateway",
"--db-url",
"sqlite::memory:",
"--drivers",
"docker,podman",
]);
assert!(!super::is_singleplayer_driver(&multi));
}

#[test]
fn file_populates_service_routing_fields() {
let _lock = ENV_LOCK
Expand Down
4 changes: 2 additions & 2 deletions crates/openshell-server/src/defaults.rs
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@ pub fn complete_local_jwt_config() -> Result<Option<GatewayJwtConfig>> {
public_key_path: paths.public_key,
kid_path: paths.kid,
gateway_id: "openshell".to_string(),
ttl_secs: 3_600,
ttl_secs: 0,
})),
_ => Err(miette::miette!(
"partial local sandbox JWT state in {}: expected jwt/signing.pem, jwt/public.pem, and jwt/kid",
Expand Down Expand Up @@ -237,6 +237,6 @@ mod tests {
assert_eq!(config.public_key_path, tmp.path().join("jwt/public.pem"));
assert_eq!(config.kid_path, tmp.path().join("jwt/kid"));
assert_eq!(config.gateway_id, "openshell");
assert_eq!(config.ttl_secs, 3_600);
assert_eq!(config.ttl_secs, 0);
}
}
52 changes: 51 additions & 1 deletion crates/openshell-server/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -705,6 +705,7 @@ async fn build_compute_runtime(
) -> Result<ComputeRuntime> {
let driver = configured_compute_driver(config)?;
info!(driver = %driver, "Using compute driver");
warn_if_kubernetes_sandbox_jwt_expiry_disabled(config, driver);

match driver {
ComputeDriverKind::Kubernetes => {
Expand Down Expand Up @@ -878,13 +879,30 @@ fn configured_compute_driver(config: &Config) -> Result<ComputeDriverKind> {
}
}

fn kubernetes_sandbox_jwt_expiry_disabled(config: &Config, driver: ComputeDriverKind) -> bool {
matches!(driver, ComputeDriverKind::Kubernetes)
&& config
.gateway_jwt
.as_ref()
.is_some_and(|jwt| jwt.ttl_secs == 0)
}

fn warn_if_kubernetes_sandbox_jwt_expiry_disabled(config: &Config, driver: ComputeDriverKind) {
if kubernetes_sandbox_jwt_expiry_disabled(config, driver) {
warn!(
"Kubernetes gateway configured with non-expiring sandbox JWTs (gateway_jwt.ttl_secs = 0); set ttl_secs > 0 for shared Kubernetes deployments"
);
}
}

#[cfg(test)]
mod tests {
use super::{
ConnectionProtocol, MultiplexService, ServerState, TlsAcceptor,
allow_plaintext_service_http, classify_initial_bytes, configured_compute_driver,
gateway_listener_addresses, is_benign_tls_handshake_failure,
kubernetes_config_for_k8s_sa_bootstrap, serve_gateway_listener,
kubernetes_config_for_k8s_sa_bootstrap, kubernetes_sandbox_jwt_expiry_disabled,
serve_gateway_listener,
};
use openshell_core::{
ComputeDriverKind, Config,
Expand Down Expand Up @@ -1288,6 +1306,38 @@ mod tests {
);
}

#[test]
fn kubernetes_sandbox_jwt_expiry_disabled_warns_only_for_kubernetes_zero_ttl() {
fn config_with_jwt_ttl(ttl_secs: u64) -> Config {
let mut config = Config::new(None);
config.gateway_jwt = Some(openshell_core::GatewayJwtConfig {
signing_key_path: "/tmp/signing.pem".into(),
public_key_path: "/tmp/public.pem".into(),
kid_path: "/tmp/kid".into(),
gateway_id: "openshell".to_string(),
ttl_secs,
});
config
}

assert!(kubernetes_sandbox_jwt_expiry_disabled(
&config_with_jwt_ttl(0),
ComputeDriverKind::Kubernetes
));
assert!(!kubernetes_sandbox_jwt_expiry_disabled(
&config_with_jwt_ttl(3600),
ComputeDriverKind::Kubernetes
));
assert!(!kubernetes_sandbox_jwt_expiry_disabled(
&config_with_jwt_ttl(0),
ComputeDriverKind::Docker
));
assert!(!kubernetes_sandbox_jwt_expiry_disabled(
&Config::new(None),
ComputeDriverKind::Kubernetes
));
}

#[test]
fn k8s_sa_bootstrap_rejects_missing_kubernetes_driver_config() {
let err = kubernetes_config_for_k8s_sa_bootstrap(None).unwrap_err();
Expand Down
Loading
Loading