diff --git a/Cargo.lock b/Cargo.lock index 88c7dc0b7c..efde222aca 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3807,6 +3807,7 @@ dependencies = [ "thiserror 2.0.18", "tokio", "tokio-stream", + "toml", "tonic", "tracing", "tracing-subscriber", diff --git a/architecture/sandbox.md b/architecture/sandbox.md index a39f699a57..ff95413421 100644 --- a/architecture/sandbox.md +++ b/architecture/sandbox.md @@ -176,6 +176,14 @@ file and builds the `Proxy-Authorization: Basic` header; a credential that is empty, contains control characters, or is not in `user:pass` form is fatal on both sides. +For Kubernetes sandboxes, the operator configures a Secret name and key rather +than a gateway-host file path. Kubernetes projects that Secret only into the +container that runs network supervision. Proxy credential Secrets require the +sidecar topology, which gives them a separate container boundary from the +workload. Combined topology is rejected because Kubernetes `fsGroup` volume +permission handling can make a shared credential mount readable by the sandbox +group. + The Basic header travels over the plain-TCP connection to the `http://` proxy, so it is readable on the network path between sandbox host and proxy. Configuring `proxy_auth_file` therefore requires the explicit opt-in diff --git a/crates/openshell-driver-kubernetes/Cargo.toml b/crates/openshell-driver-kubernetes/Cargo.toml index 2c02f864ab..f9f5bba398 100644 --- a/crates/openshell-driver-kubernetes/Cargo.toml +++ b/crates/openshell-driver-kubernetes/Cargo.toml @@ -37,6 +37,7 @@ miette = { workspace = true } [dev-dependencies] temp-env = "0.3" +toml = { workspace = true } [lints] workspace = true diff --git a/crates/openshell-driver-kubernetes/src/config.rs b/crates/openshell-driver-kubernetes/src/config.rs index 5311f56436..fb38cd2135 100644 --- a/crates/openshell-driver-kubernetes/src/config.rs +++ b/crates/openshell-driver-kubernetes/src/config.rs @@ -253,6 +253,25 @@ pub struct KubernetesComputeConfig { pub topology: SupervisorTopology, /// Sidecar-only settings used when `topology = "sidecar"`. pub sidecar: KubernetesSidecarConfig, + /// Corporate HTTP forward proxy used by the network supervisor for + /// policy-approved TLS CONNECT egress. + pub https_proxy: Option, + /// Comma-separated destinations that bypass the corporate proxy while + /// continuing through `OpenShell` policy evaluation. + pub no_proxy: Option, + /// Name of the Kubernetes Secret holding the `user:pass` proxy credential. + /// The Secret is mounted only in the network-supervising container. The + /// driver validates this reference at startup; the supervisor validates + /// the Secret content when kubelet mounts it before accepting egress. + pub proxy_auth_secret_name: Option, + /// Key in `proxy_auth_secret_name` containing the `user:pass` credential. + pub proxy_auth_secret_key: Option, + /// Explicit acknowledgement that Basic authentication is cleartext over + /// the connection to an `http://` forward proxy. + pub proxy_auth_allow_insecure: Option, + /// Send hostnames rather than validated IPs in CONNECT requests. This is a + /// last-resort compatibility mode for hostname-filtering proxy ACLs. + pub proxy_connect_by_hostname: Option, pub grpc_endpoint: String, pub ssh_socket_path: String, pub client_tls_secret_name: String, @@ -346,6 +365,12 @@ impl Default for KubernetesComputeConfig { supervisor_sideload_method: SupervisorSideloadMethod::default(), topology: SupervisorTopology::default(), sidecar: KubernetesSidecarConfig::default(), + https_proxy: None, + no_proxy: None, + proxy_auth_secret_name: None, + proxy_auth_secret_key: None, + proxy_auth_allow_insecure: None, + proxy_connect_by_hostname: None, grpc_endpoint: String::new(), ssh_socket_path: openshell_core::container_paths::SSH_SOCKET_PATH.to_string(), client_tls_secret_name: String::new(), @@ -395,6 +420,88 @@ impl KubernetesComputeConfig { self.sidecar.validate_proxy_uid() } + /// Validate the operator-owned corporate upstream proxy configuration. + pub fn validate_upstream_proxy_config(&self) -> Result<(), String> { + use openshell_core::driver_utils::{UpstreamProxyUrlError, parse_upstream_proxy_url}; + + if let Some(url) = &self.https_proxy { + parse_upstream_proxy_url(url).map_err(|err| match err { + UpstreamProxyUrlError::Empty => "https_proxy must not be empty when set".to_string(), + UpstreamProxyUrlError::InlineCredentials => "https_proxy must not embed credentials in the URL; supply them through proxy_auth_secret_name and proxy_auth_secret_key".to_string(), + err => format!("https_proxy {err}"), + })?; + } + + if let Some(list) = self.no_proxy.as_deref() { + if list.trim().is_empty() { + return Err("no_proxy must not be empty when set; omit it instead".to_string()); + } + if self.https_proxy.is_none() { + return Err("no_proxy is set but no https_proxy is configured".to_string()); + } + } + + let secret_name = self.proxy_auth_secret_name.as_deref(); + let secret_key = self.proxy_auth_secret_key.as_deref(); + match (secret_name, secret_key) { + (None, None) => { + if self.proxy_auth_allow_insecure == Some(true) { + return Err("proxy_auth_allow_insecure is set but no proxy credential Secret is configured".to_string()); + } + } + (Some(name), Some(key)) => { + if name.trim().is_empty() || key.trim().is_empty() { + return Err( + "proxy credential Secret name and key must not be empty".to_string() + ); + } + if !is_dns1123_subdomain(name) { + return Err( + "proxy_auth_secret_name must be a valid Kubernetes DNS-1123 subdomain" + .to_string(), + ); + } + if !key + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'-' | b'_')) + { + return Err( + "proxy_auth_secret_key must contain only letters, digits, '.', '-', or '_'" + .to_string(), + ); + } + if self.https_proxy.is_none() { + return Err( + "proxy credential Secret is set but no https_proxy is configured" + .to_string(), + ); + } + if self.proxy_auth_allow_insecure != Some(true) { + return Err("proxy credentials use cleartext Basic auth over the connection to the http:// proxy; set proxy_auth_allow_insecure = true to accept that exposure, or remove the credential Secret".to_string()); + } + if self.topology == SupervisorTopology::Combined { + return Err( + "proxy credential Secrets require topology = \"sidecar\"; combined topology shares the credential mount with the workload and fsGroup can make it readable by the sandbox user" + .to_string(), + ); + } + } + _ => { + return Err( + "proxy_auth_secret_name and proxy_auth_secret_key must be set together" + .to_string(), + ); + } + } + + if self.proxy_connect_by_hostname.is_some() && self.https_proxy.is_none() { + return Err( + "proxy_connect_by_hostname is set but no https_proxy is configured".to_string(), + ); + } + Ok(()) + } + /// Resolve the sandbox UID/GID pair. /// /// Resolution order: @@ -475,6 +582,20 @@ impl KubernetesComputeConfig { } } +fn is_dns1123_subdomain(value: &str) -> bool { + !value.is_empty() + && value.len() <= 253 + && value.split('.').all(|label| { + !label.is_empty() + && label.len() <= 63 + && !label.starts_with('-') + && !label.ends_with('-') + && label + .bytes() + .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'-') + }) +} + fn validate_provider_spiffe_workload_api_socket_path_value( socket_path: &str, ) -> Result<(), String> { @@ -966,4 +1087,126 @@ mod tests { let uid = cfg.resolve_sandbox_uid(None); assert_eq!(cfg.resolve_sandbox_gid(uid, None), uid); } + + #[test] + fn upstream_proxy_config_accepts_http_proxy_without_credentials() { + let cfg = KubernetesComputeConfig { + https_proxy: Some("http://proxy.corp.example:8080".to_string()), + no_proxy: Some(".svc.cluster.local,10.96.0.0/12".to_string()), + ..KubernetesComputeConfig::default() + }; + assert!(cfg.validate_upstream_proxy_config().is_ok()); + } + + #[test] + fn upstream_proxy_config_accepts_secret_credentials_with_acknowledgement() { + let cfg = KubernetesComputeConfig { + topology: SupervisorTopology::Sidecar, + https_proxy: Some("http://proxy.corp.example:8080".to_string()), + proxy_auth_secret_name: Some("corporate-proxy-auth".to_string()), + proxy_auth_secret_key: Some("credentials".to_string()), + proxy_auth_allow_insecure: Some(true), + ..KubernetesComputeConfig::default() + }; + assert!(cfg.validate_upstream_proxy_config().is_ok()); + } + + #[test] + fn toml_deserializes_sidecar_upstream_proxy_settings() { + let cfg: KubernetesComputeConfig = toml::from_str( + r#" + topology = "sidecar" + https_proxy = "http://proxy.corp.example:8080" + no_proxy = ".svc.cluster.local,10.96.0.0/12" + proxy_auth_secret_name = "corporate-proxy-auth" + proxy_auth_secret_key = "credentials" + proxy_auth_allow_insecure = true + proxy_connect_by_hostname = true + "#, + ) + .unwrap(); + assert!(cfg.validate_upstream_proxy_config().is_ok()); + assert_eq!( + cfg.https_proxy.as_deref(), + Some("http://proxy.corp.example:8080") + ); + assert_eq!( + cfg.proxy_auth_secret_name.as_deref(), + Some("corporate-proxy-auth") + ); + } + + #[test] + fn upstream_proxy_config_rejects_incoherent_auxiliary_settings() { + for cfg in [ + KubernetesComputeConfig { + no_proxy: Some(".svc".to_string()), + ..KubernetesComputeConfig::default() + }, + KubernetesComputeConfig { + https_proxy: Some("http://proxy.corp.example:8080".to_string()), + proxy_auth_secret_name: Some("corporate-proxy-auth".to_string()), + ..KubernetesComputeConfig::default() + }, + KubernetesComputeConfig { + https_proxy: Some("http://proxy.corp.example:8080".to_string()), + proxy_auth_secret_name: Some("corporate-proxy-auth".to_string()), + proxy_auth_secret_key: Some("credentials".to_string()), + ..KubernetesComputeConfig::default() + }, + KubernetesComputeConfig { + proxy_connect_by_hostname: Some(true), + ..KubernetesComputeConfig::default() + }, + ] { + assert!(cfg.validate_upstream_proxy_config().is_err()); + } + } + + #[test] + fn upstream_proxy_config_rejects_unsupported_proxy_scheme() { + let cfg = KubernetesComputeConfig { + https_proxy: Some("https://proxy.corp.example:8443".to_string()), + ..KubernetesComputeConfig::default() + }; + let err = cfg.validate_upstream_proxy_config().unwrap_err(); + assert!(err.contains("https_proxy"), "{err}"); + } + + #[test] + fn upstream_proxy_config_rejects_invalid_secret_name() { + let cfg = KubernetesComputeConfig { + https_proxy: Some("http://proxy.corp.example:8080".to_string()), + proxy_auth_secret_name: Some("Not_A_Secret".to_string()), + proxy_auth_secret_key: Some("credentials".to_string()), + proxy_auth_allow_insecure: Some(true), + ..KubernetesComputeConfig::default() + }; + let err = cfg.validate_upstream_proxy_config().unwrap_err(); + assert!(err.contains("proxy_auth_secret_name"), "{err}"); + } + + #[test] + fn upstream_proxy_config_rejects_credentials_in_combined_topology() { + let cfg = KubernetesComputeConfig { + topology: SupervisorTopology::Combined, + https_proxy: Some("http://proxy.corp.example:8080".to_string()), + proxy_auth_secret_name: Some("corporate-proxy-auth".to_string()), + proxy_auth_secret_key: Some("credentials".to_string()), + proxy_auth_allow_insecure: Some(true), + ..KubernetesComputeConfig::default() + }; + let err = cfg.validate_upstream_proxy_config().unwrap_err(); + assert!(err.contains("topology = \"sidecar\""), "{err}"); + } + + #[test] + fn upstream_proxy_config_allows_explicit_false_acknowledgement_without_credentials() { + let cfg = KubernetesComputeConfig { + https_proxy: Some("http://proxy.corp.example:8080".to_string()), + proxy_auth_allow_insecure: Some(false), + ..KubernetesComputeConfig::default() + }; + assert!(cfg.validate_upstream_proxy_config().is_ok()); + } } diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index 2f1ea72a32..e334c3c55e 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -246,11 +246,13 @@ impl From<&KubernetesDriverVolumeMountConfig> for VolumeMount { } const CLIENT_TLS_VOLUME_NAME: &str = "openshell-client-tls"; +const UPSTREAM_PROXY_AUTH_VOLUME_NAME: &str = "openshell-upstream-proxy-auth"; const SERVICE_ACCOUNT_TOKEN_VOLUME_NAME: &str = "openshell-sa-token"; const SERVICE_ACCOUNT_TOKEN_MOUNT_PATH: &str = "/var/run/secrets/openshell"; const KUBERNETES_DRIVER_RESERVED_VOLUME_NAMES: &[&str] = &[ CLIENT_TLS_VOLUME_NAME, + UPSTREAM_PROXY_AUTH_VOLUME_NAME, SERVICE_ACCOUNT_TOKEN_VOLUME_NAME, SPIFFE_WORKLOAD_API_VOLUME_NAME, SUPERVISOR_VOLUME_NAME, @@ -459,6 +461,9 @@ impl KubernetesComputeDriver { config .validate_proxy_uid() .map_err(KubernetesDriverError::Precondition)?; + config + .validate_upstream_proxy_config() + .map_err(KubernetesDriverError::Precondition)?; let base_config = match kube::Config::incluster() { Ok(c) => c, Err(_) => kube::Config::infer() @@ -842,6 +847,12 @@ impl KubernetesComputeDriver { .config .sidecar .process_binary_aware_network_policy, + https_proxy: self.config.https_proxy.as_deref(), + no_proxy: self.config.no_proxy.as_deref(), + proxy_auth_secret_name: self.config.proxy_auth_secret_name.as_deref(), + proxy_auth_secret_key: self.config.proxy_auth_secret_key.as_deref(), + proxy_auth_allow_insecure: self.config.proxy_auth_allow_insecure == Some(true), + proxy_connect_by_hostname: self.config.proxy_connect_by_hostname == Some(true), service_account_name: &self.config.service_account_name, sandbox_id: &sandbox.id, sandbox_name: &sandbox.name, @@ -1573,19 +1584,20 @@ fn apply_supervisor_binary_source( /// side-loaded binary as root so it can create network namespaces, set up the /// proxy, and configure Landlock/seccomp. #[allow(clippy::similar_names)] -fn apply_supervisor_sideload( +fn apply_supervisor_sideload_with_params( pod_template: &mut serde_json::Value, - supervisor_image: &str, - supervisor_image_pull_policy: &str, - method: SupervisorSideloadMethod, - sandbox_uid: u32, - sandbox_gid: u32, + params: &SandboxPodParams<'_>, ) { let Some(spec) = pod_template.get_mut("spec").and_then(|v| v.as_object_mut()) else { return; }; - apply_supervisor_binary_source(spec, supervisor_image, supervisor_image_pull_policy, method); + apply_supervisor_binary_source( + spec, + params.supervisor_image, + params.supervisor_image_pull_policy, + params.supervisor_sideload_method, + ); // Find the agent container and add volume mount + command override let Some(containers) = spec.get_mut("containers").and_then(|v| v.as_array_mut()) else { @@ -1603,14 +1615,13 @@ fn apply_supervisor_sideload( if let Some(container) = containers.get_mut(index).and_then(|v| v.as_object_mut()) { // Override command to use the side-loaded supervisor binary - container.insert( - "command".to_string(), - serde_json::json!([ - format!("{}/openshell-sandbox", SUPERVISOR_MOUNT_PATH), - "--workdir", - driver_mounts::DEFAULT_WORKSPACE_ROOT - ]), - ); + let mut command = vec![ + format!("{}/openshell-sandbox", SUPERVISOR_MOUNT_PATH), + "--workdir".to_string(), + driver_mounts::DEFAULT_WORKSPACE_ROOT.to_string(), + ]; + command.extend(upstream_proxy_cli_args(params)); + container.insert("command".to_string(), serde_json::json!(command)); // Force the supervisor to run as root (UID 0). Sandbox images may set // a non-root USER directive (e.g. `USER sandbox`), but the supervisor @@ -1641,9 +1652,88 @@ fn apply_supervisor_sideload( .or_insert_with(|| serde_json::json!([])) .as_array_mut(); if let Some(env) = env { - apply_resolved_identity_env(env, sandbox_uid, sandbox_gid); + apply_resolved_identity_env(env, params.sandbox_uid, params.sandbox_gid); } + if has_upstream_proxy_credentials(params) { + let volume_mounts = container + .entry("volumeMounts") + .or_insert_with(|| serde_json::json!([])) + .as_array_mut(); + if let Some(volume_mounts) = volume_mounts { + volume_mounts.push(upstream_proxy_auth_volume_mount()); + } + } + } +} + +#[cfg(test)] +#[allow(clippy::similar_names)] +fn apply_supervisor_sideload( + pod_template: &mut serde_json::Value, + supervisor_image: &str, + supervisor_image_pull_policy: &str, + method: SupervisorSideloadMethod, + sandbox_uid: u32, + sandbox_gid: u32, +) { + let params = SandboxPodParams { + supervisor_image, + supervisor_image_pull_policy, + supervisor_sideload_method: method, + sandbox_uid, + sandbox_gid, + ..SandboxPodParams::default() + }; + apply_supervisor_sideload_with_params(pod_template, ¶ms); +} + +fn upstream_proxy_cli_args(params: &SandboxPodParams<'_>) -> Vec { + let mut args = Vec::new(); + if let Some(url) = params.https_proxy { + args.extend(["--upstream-proxy".to_string(), url.to_string()]); + } + if let Some(list) = params.no_proxy { + args.extend(["--upstream-no-proxy".to_string(), list.to_string()]); } + if has_upstream_proxy_credentials(params) { + args.extend([ + "--upstream-proxy-auth-file".to_string(), + openshell_core::container_paths::UPSTREAM_PROXY_AUTH_MOUNT_PATH.to_string(), + ]); + } + if params.proxy_auth_allow_insecure { + args.push("--upstream-proxy-auth-allow-insecure".to_string()); + } + if params.proxy_connect_by_hostname { + args.push("--upstream-proxy-connect-by-hostname".to_string()); + } + args +} + +fn upstream_proxy_auth_volume_mount() -> serde_json::Value { + serde_json::json!({ + "name": UPSTREAM_PROXY_AUTH_VOLUME_NAME, + "mountPath": upstream_proxy_auth_volume_mount_path(), + "readOnly": true, + }) +} + +fn upstream_proxy_auth_volume_mount_path() -> &'static str { + Path::new(openshell_core::container_paths::UPSTREAM_PROXY_AUTH_MOUNT_PATH) + .parent() + .and_then(Path::to_str) + .expect("upstream proxy auth path has a parent directory") +} + +fn upstream_proxy_auth_file_name() -> &'static str { + Path::new(openshell_core::container_paths::UPSTREAM_PROXY_AUTH_MOUNT_PATH) + .file_name() + .and_then(|name| name.to_str()) + .expect("upstream proxy auth path has a UTF-8 file name") +} + +fn has_upstream_proxy_credentials(params: &SandboxPodParams<'_>) -> bool { + params.proxy_auth_secret_name.is_some() && params.proxy_auth_secret_key.is_some() } fn sidecar_state_volume_mount() -> serde_json::Value { @@ -1783,6 +1873,14 @@ fn supervisor_sidecar_container( } ] }); + container["command"] + .as_array_mut() + .expect("network supervisor command is an array") + .extend( + upstream_proxy_cli_args(params) + .into_iter() + .map(serde_json::Value::String), + ); if !params.supervisor_image_pull_policy.is_empty() { container["imagePullPolicy"] = serde_json::json!(params.supervisor_image_pull_policy); } @@ -1796,6 +1894,12 @@ fn supervisor_sidecar_container( "readOnly": true, })); } + if has_upstream_proxy_credentials(params) { + container["volumeMounts"] + .as_array_mut() + .expect("volumeMounts is an array") + .push(upstream_proxy_auth_volume_mount()); + } if let Some(profile) = params.app_armor_profile { container["securityContext"]["appArmorProfile"] = app_armor_profile_to_k8s(profile); } @@ -2176,6 +2280,7 @@ fn default_workspace_volume_claim_templates( } /// Parameters shared by `sandbox_to_k8s_spec` and `sandbox_template_to_k8s`. +#[allow(clippy::struct_excessive_bools)] struct SandboxPodParams<'a> { default_image: &'a str, image_pull_policy: &'a str, @@ -2186,6 +2291,12 @@ struct SandboxPodParams<'a> { topology: SupervisorTopology, proxy_uid: u32, process_binary_aware_network_policy: bool, + https_proxy: Option<&'a str>, + no_proxy: Option<&'a str>, + proxy_auth_secret_name: Option<&'a str>, + proxy_auth_secret_key: Option<&'a str>, + proxy_auth_allow_insecure: bool, + proxy_connect_by_hostname: bool, service_account_name: &'a str, sandbox_id: &'a str, sandbox_name: &'a str, @@ -2221,6 +2332,12 @@ impl Default for SandboxPodParams<'_> { topology: SupervisorTopology::default(), proxy_uid: DEFAULT_PROXY_UID, process_binary_aware_network_policy: true, + https_proxy: None, + no_proxy: None, + proxy_auth_secret_name: None, + proxy_auth_secret_key: None, + proxy_auth_allow_insecure: false, + proxy_connect_by_hostname: false, service_account_name: DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, sandbox_id: "", sandbox_name: "", @@ -2626,6 +2743,32 @@ fn sandbox_template_to_k8s_with_validated_config( } })); } + if has_upstream_proxy_credentials(params) { + let secret_name = params + .proxy_auth_secret_name + .expect("complete proxy credential reference has a Secret name"); + let secret_key = params + .proxy_auth_secret_key + .expect("complete proxy credential reference has a Secret key"); + // The credential volume is mounted only into the container that runs + // network supervision. Sidecar mode uses the pod fsGroup already + // required for its non-root network supervisor. + let default_mode = match params.topology { + SupervisorTopology::Combined => 0o400, + SupervisorTopology::Sidecar => 0o440, + }; + volumes.push(serde_json::json!({ + "name": UPSTREAM_PROXY_AUTH_VOLUME_NAME, + "secret": { + "secretName": secret_name, + "defaultMode": default_mode, + "items": [{ + "key": secret_key, + "path": upstream_proxy_auth_file_name(), + }] + } + })); + } if params.provider_spiffe_enabled { volumes.push(serde_json::json!({ "name": SPIFFE_WORKLOAD_API_VOLUME_NAME, @@ -2686,14 +2829,7 @@ fn sandbox_template_to_k8s_with_validated_config( match params.topology { SupervisorTopology::Combined => { - apply_supervisor_sideload( - &mut result, - params.supervisor_image, - params.supervisor_image_pull_policy, - params.supervisor_sideload_method, - params.sandbox_uid, - params.sandbox_gid, - ); + apply_supervisor_sideload_with_params(&mut result, params); } SupervisorTopology::Sidecar => { apply_supervisor_sidecar_topology( @@ -6068,4 +6204,101 @@ mod tests { .is_none() ); } + + #[test] + fn upstream_proxy_is_injected_only_into_network_supervisors() { + let params = SandboxPodParams { + topology: SupervisorTopology::Sidecar, + supervisor_sideload_method: SupervisorSideloadMethod::InitContainer, + supervisor_image: "supervisor-image:latest", + https_proxy: Some("http://proxy.corp.example:8080"), + no_proxy: Some(".svc.cluster.local,10.96.0.0/12"), + proxy_auth_secret_name: Some("corporate-proxy-auth"), + proxy_auth_secret_key: Some("credentials"), + proxy_auth_allow_insecure: true, + proxy_connect_by_hostname: true, + sandbox_uid: 1500, + sandbox_gid: 1500, + ..SandboxPodParams::default() + }; + let pod = sandbox_template_to_k8s( + &SandboxTemplate::default(), + false, + &std::collections::HashMap::new(), + false, + ¶ms, + ); + let containers = pod["spec"]["containers"].as_array().unwrap(); + let network = containers + .iter() + .find(|container| container["name"] == SUPERVISOR_NETWORK_SIDECAR_NAME) + .unwrap(); + let command = network["command"].as_array().unwrap(); + assert!(command.iter().any(|arg| arg == "--upstream-proxy")); + assert!(command.iter().any(|arg| arg == "--upstream-no-proxy")); + let auth_file_index = command + .iter() + .position(|arg| arg == "--upstream-proxy-auth-file") + .unwrap(); + assert_eq!( + command[auth_file_index + 1], + openshell_core::container_paths::UPSTREAM_PROXY_AUTH_MOUNT_PATH + ); + assert!( + command + .iter() + .any(|arg| arg == "--upstream-proxy-auth-allow-insecure") + ); + assert!( + command + .iter() + .any(|arg| arg == "--upstream-proxy-connect-by-hostname") + ); + assert!( + network["volumeMounts"] + .as_array() + .unwrap() + .iter() + .any(|mount| mount["name"] == UPSTREAM_PROXY_AUTH_VOLUME_NAME) + ); + + let init = pod["spec"]["initContainers"] + .as_array() + .unwrap() + .iter() + .find(|container| container["name"] == SUPERVISOR_NETWORK_INIT_CONTAINER_NAME) + .unwrap(); + assert!(!init["command"].as_array().unwrap().iter().any(|arg| { + arg.as_str() + .is_some_and(|arg| arg.starts_with("--upstream-")) + })); + let agent = containers + .iter() + .find(|container| container["name"] == "agent") + .unwrap(); + assert!( + !agent["volumeMounts"] + .as_array() + .unwrap() + .iter() + .any(|mount| mount["name"] == UPSTREAM_PROXY_AUTH_VOLUME_NAME) + ); + assert!(!agent["env"].as_array().unwrap().iter().any(|entry| { + entry["value"] == "corporate-proxy-auth" || entry["value"] == "credentials" + })); + + let volume = pod["spec"]["volumes"] + .as_array() + .unwrap() + .iter() + .find(|volume| volume["name"] == UPSTREAM_PROXY_AUTH_VOLUME_NAME) + .unwrap(); + assert_eq!(volume["secret"]["secretName"], "corporate-proxy-auth"); + assert_eq!(volume["secret"]["items"][0]["key"], "credentials"); + assert_eq!( + volume["secret"]["items"][0]["path"], + upstream_proxy_auth_file_name() + ); + assert_eq!(volume["secret"]["defaultMode"], 0o440); + } } diff --git a/crates/openshell-driver-kubernetes/src/main.rs b/crates/openshell-driver-kubernetes/src/main.rs index b7d5514ac2..99df4ea165 100644 --- a/crates/openshell-driver-kubernetes/src/main.rs +++ b/crates/openshell-driver-kubernetes/src/main.rs @@ -18,6 +18,7 @@ use openshell_driver_kubernetes::{ #[derive(Parser, Debug)] #[command(name = "openshell-driver-kubernetes")] #[command(version = VERSION)] +#[allow(clippy::struct_excessive_bools)] struct Args { #[arg( long, @@ -100,6 +101,30 @@ struct Args { )] sidecar_process_binary_aware_network_policy: bool, + /// Corporate HTTP forward proxy for policy-approved TLS CONNECT egress. + #[arg(long, env = "OPENSHELL_UPSTREAM_PROXY")] + https_proxy: Option, + + /// Comma-separated destinations that bypass the corporate proxy. + #[arg(long, env = "OPENSHELL_UPSTREAM_NO_PROXY")] + no_proxy: Option, + + /// Kubernetes Secret name containing the upstream proxy credential. + #[arg(long, env = "OPENSHELL_UPSTREAM_PROXY_AUTH_SECRET_NAME")] + proxy_auth_secret_name: Option, + + /// Kubernetes Secret key containing the upstream proxy credential. + #[arg(long, env = "OPENSHELL_UPSTREAM_PROXY_AUTH_SECRET_KEY")] + proxy_auth_secret_key: Option, + + /// Acknowledge cleartext Basic auth to an http:// upstream proxy. + #[arg(long, env = "OPENSHELL_UPSTREAM_PROXY_AUTH_ALLOW_INSECURE", action = ArgAction::SetTrue)] + proxy_auth_allow_insecure: bool, + + /// Send destination hostnames rather than validated IPs in CONNECT. + #[arg(long, env = "OPENSHELL_UPSTREAM_PROXY_CONNECT_BY_HOSTNAME", action = ArgAction::SetTrue)] + proxy_connect_by_hostname: bool, + #[arg(long, env = "OPENSHELL_ENABLE_USER_NAMESPACES")] enable_user_namespaces: bool, @@ -148,6 +173,12 @@ async fn main() -> Result<()> { proxy_uid: args.sidecar_proxy_uid, process_binary_aware_network_policy: args.sidecar_process_binary_aware_network_policy, }, + https_proxy: args.https_proxy, + no_proxy: args.no_proxy, + proxy_auth_secret_name: args.proxy_auth_secret_name, + proxy_auth_secret_key: args.proxy_auth_secret_key, + proxy_auth_allow_insecure: args.proxy_auth_allow_insecure.then_some(true), + proxy_connect_by_hostname: args.proxy_connect_by_hostname.then_some(true), grpc_endpoint: args.grpc_endpoint.unwrap_or_default(), ssh_socket_path: args.sandbox_ssh_socket_path, client_tls_secret_name: args.client_tls_secret_name.unwrap_or_default(), diff --git a/deploy/helm/openshell/README.md b/deploy/helm/openshell/README.md index 7096a8ca74..8e0e88ba64 100644 --- a/deploy/helm/openshell/README.md +++ b/deploy/helm/openshell/README.md @@ -273,6 +273,13 @@ add `ci/values-spire.yaml` to the OpenShell release values files. | supervisor.sideloadMethod | string | `""` | How the supervisor binary is delivered into sandbox pods. Empty (default) = auto-detect from cluster version: K8s >= v1.35 -> "image-volume" (ImageVolume enabled by default; GA in v1.36) K8s < v1.35 -> "init-container" (copies via init container + emptyDir) On K8s v1.33-v1.34 with the ImageVolume feature gate manually enabled, set this to "image-volume" explicitly. | | supervisor.topology | string | `"combined"` | Supervisor pod topology for Kubernetes sandboxes. "combined" runs the current single supervisor container in the agent pod. "sidecar" runs network enforcement in a dedicated sidecar and the process supervisor as a low-capability wrapper in the agent container. | | tolerations | list | `[]` | Tolerations for the gateway pod. | +| upstreamProxy | object | `{"authAllowInsecure":false,"authSecret":{"key":"","name":""},"connectByHostname":false,"noProxy":"","url":""}` | Operator-owned corporate forward proxy for policy-approved TLS egress from Kubernetes sandboxes. The workload cannot select or override it. | +| upstreamProxy.authAllowInsecure | bool | `false` | Required when authSecret is configured because Basic auth to an HTTP proxy is cleartext. | +| upstreamProxy.authSecret.key | string | `""` | Secret key containing the proxy credential. | +| upstreamProxy.authSecret.name | string | `""` | Existing Secret in the sandbox namespace containing a user:pass value. | +| upstreamProxy.connectByHostname | bool | `false` | Last-resort option for hostname-filtering proxy ACLs. It lets the proxy resolve CONNECT targets. | +| upstreamProxy.noProxy | string | `""` | Comma-separated destinations that bypass only the corporate proxy. | +| upstreamProxy.url | string | `""` | HTTP proxy URL in http://host:port form. HTTPS-to-proxy is not supported. | | workload.allowMultiReplicaStatefulSet | bool | `false` | Allow replicaCount > 1 while rendering a StatefulSet. Prefer workload.kind=deployment for external database-backed multi-replica gateways; this override exists for operators who explicitly require StatefulSet identity or storage semantics. | | workload.kind | string | `"statefulset"` | Gateway workload controller kind. Use `statefulset` for the default SQLite database, or `deployment` when server.externalDbSecret points at an external database. | diff --git a/deploy/helm/openshell/ci/values-corporate-proxy-e2e.yaml b/deploy/helm/openshell/ci/values-corporate-proxy-e2e.yaml new file mode 100644 index 0000000000..35532440b2 --- /dev/null +++ b/deploy/helm/openshell/ci/values-corporate-proxy-e2e.yaml @@ -0,0 +1,13 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# The Kubernetes corporate-proxy e2e wrapper supplies the generated proxy URL +# and creates `openshell-e2e-proxy-auth` before Helm installs the gateway. +supervisor: + topology: sidecar + +upstreamProxy: + authSecret: + name: openshell-e2e-proxy-auth + key: proxy-auth + authAllowInsecure: true diff --git a/deploy/helm/openshell/templates/gateway-config.yaml b/deploy/helm/openshell/templates/gateway-config.yaml index e22b5e7485..fdcf79a2d6 100644 --- a/deploy/helm/openshell/templates/gateway-config.yaml +++ b/deploy/helm/openshell/templates/gateway-config.yaml @@ -132,6 +132,24 @@ data: supervisor_sideload_method = {{ include "openshell.supervisorSideloadMethod" . | quote }} topology = {{ .Values.supervisor.topology | default "combined" | quote }} sa_token_ttl_secs = {{ .Values.server.sandboxJwt.k8sSaTokenTtlSecs | default 3600 }} + {{- if .Values.upstreamProxy.url }} + https_proxy = {{ .Values.upstreamProxy.url | quote }} + {{- end }} + {{- if .Values.upstreamProxy.noProxy }} + no_proxy = {{ .Values.upstreamProxy.noProxy | quote }} + {{- end }} + {{- if .Values.upstreamProxy.authSecret.name }} + proxy_auth_secret_name = {{ .Values.upstreamProxy.authSecret.name | quote }} + {{- end }} + {{- if .Values.upstreamProxy.authSecret.key }} + proxy_auth_secret_key = {{ .Values.upstreamProxy.authSecret.key | quote }} + {{- end }} + {{- if and .Values.upstreamProxy.authSecret.name .Values.upstreamProxy.authSecret.key }} + proxy_auth_allow_insecure = {{ .Values.upstreamProxy.authAllowInsecure }} + {{- end }} + {{- if .Values.upstreamProxy.connectByHostname }} + proxy_connect_by_hostname = true + {{- end }} {{- if .Values.server.providerTokenGrants.spiffe.enabled }} provider_spiffe_workload_api_socket_path = {{ .Values.server.providerTokenGrants.spiffe.workloadApiSocketPath | quote }} {{- end }} diff --git a/deploy/helm/openshell/tests/gateway_config_test.yaml b/deploy/helm/openshell/tests/gateway_config_test.yaml index f98c321fee..01098955aa 100644 --- a/deploy/helm/openshell/tests/gateway_config_test.yaml +++ b/deploy/helm/openshell/tests/gateway_config_test.yaml @@ -93,6 +93,35 @@ tests: path: data["gateway.toml"] pattern: 'supervisor[_]topology\s*=' + - it: renders operator-owned upstream proxy settings under the Kubernetes driver + template: templates/gateway-config.yaml + set: + upstreamProxy.url: http://proxy.corp.example:8080 + upstreamProxy.noProxy: .svc.cluster.local,10.96.0.0/12 + upstreamProxy.authSecret.name: corporate-proxy-auth + upstreamProxy.authSecret.key: credentials + upstreamProxy.authAllowInsecure: true + upstreamProxy.connectByHostname: true + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: '(?ms)\[openshell\.drivers\.kubernetes\].*?https_proxy\s*=\s*"http://proxy\.corp\.example:8080"' + - matchRegex: + path: data["gateway.toml"] + pattern: 'proxy_auth_secret_name\s*=\s*"corporate-proxy-auth"' + - matchRegex: + path: data["gateway.toml"] + pattern: 'proxy_auth_secret_key\s*=\s*"credentials"' + - matchRegex: + path: data["gateway.toml"] + pattern: 'proxy_auth_allow_insecure\s*=\s*true' + - matchRegex: + path: data["gateway.toml"] + pattern: 'no_proxy\s*=\s*"\.svc\.cluster\.local,10\.96\.0\.0/12"' + - matchRegex: + path: data["gateway.toml"] + pattern: 'proxy_connect_by_hostname\s*=\s*true' + - it: uses the gateway built-in supervisor image by default template: templates/gateway-config.yaml asserts: diff --git a/deploy/helm/openshell/values.yaml b/deploy/helm/openshell/values.yaml index 39205df1bf..7feaf09149 100644 --- a/deploy/helm/openshell/values.yaml +++ b/deploy/helm/openshell/values.yaml @@ -62,6 +62,23 @@ supervisor: # policy.binaries. processBinaryAwareNetworkPolicy: true +# -- Operator-owned corporate forward proxy for policy-approved TLS egress +# from Kubernetes sandboxes. The workload cannot select or override it. +upstreamProxy: + # -- HTTP proxy URL in http://host:port form. HTTPS-to-proxy is not supported. + url: "" + # -- Comma-separated destinations that bypass only the corporate proxy. + noProxy: "" + authSecret: + # -- Existing Secret in the sandbox namespace containing a user:pass value. + name: "" + # -- Secret key containing the proxy credential. + key: "" + # -- Required when authSecret is configured because Basic auth to an HTTP proxy is cleartext. + authAllowInsecure: false + # -- Last-resort option for hostname-filtering proxy ACLs. It lets the proxy resolve CONNECT targets. + connectByHostname: false + # -- Image pull secrets attached to gateway and helper pods. imagePullSecrets: [] # -- Override the chart name used in generated resource names. diff --git a/docs/kubernetes/setup.mdx b/docs/kubernetes/setup.mdx index c2fca827f1..5d316cf81a 100644 --- a/docs/kubernetes/setup.mdx +++ b/docs/kubernetes/setup.mdx @@ -162,6 +162,7 @@ The most commonly changed values are: | `supervisor.sideloadMethod` | How the supervisor binary is delivered into sandbox pods. Leave empty to auto-detect based on cluster version: clusters running Kubernetes 1.35 or later use `image-volume` (ImageVolume GA in 1.36); older clusters use `init-container`. Set explicitly to `image-volume` on Kubernetes 1.33 or 1.34 with the ImageVolume feature gate enabled, or to `init-container` to force the legacy path on any version. | | `supervisor.topology` | Sandbox pod topology. Refer to [Topology](/kubernetes/topology). | | `supervisor.sidecar.proxyUid` | Non-root UID used when sidecar process/binary-aware network policy is disabled. The default binary-aware sidecar runs as UID 0 instead. The configured UID must not match the sandbox UID. | +| `upstreamProxy` | Operator-owned corporate HTTP forward proxy for policy-approved TLS egress. Refer to [Configure a Corporate Upstream Proxy](#configure-a-corporate-upstream-proxy). | Use a values file for repeatable deployments: @@ -197,6 +198,36 @@ server: - name: regcred ``` +## Configure a Corporate Upstream Proxy + +Configure a corporate forward proxy when sandbox TLS egress cannot dial the Internet directly. OpenShell evaluates policy and SSRF checks before it opens an HTTP CONNECT tunnel through the proxy. The proxy URL is operator-owned configuration. Sandbox environment variables cannot select, replace, or bypass it. + +Create the credential Secret in the sandbox namespace when the proxy requires Basic authentication. The Secret value uses the `user:pass` form. + +```shell +kubectl -n openshell create secret generic corporate-proxy-auth \ + --from-literal=credentials="$PROXY_USER:$PROXY_PASSWORD" +``` + +Add the proxy settings to your Helm values file. Replace the DNS suffixes and CIDRs in `noProxy` with values for your cluster. `noProxy` bypasses only the corporate proxy. OpenShell policy evaluation still applies. + +```yaml +upstreamProxy: + url: http://proxy.corp.example:8080 + noProxy: .svc,.svc.cluster.local,10.96.0.0/12,10.244.0.0/16 + authSecret: + name: corporate-proxy-auth + key: credentials + authAllowInsecure: true + +supervisor: + topology: sidecar +``` + +Use `authAllowInsecure: true` only when you accept that Basic authentication is cleartext on the connection to an `http://` proxy. The initial release supports `http://` proxy endpoints and TLS CONNECT egress. It does not support HTTPS-to-proxy, custom corporate CA bundles, or forwarding plain HTTP egress through the proxy. + +Proxy credentials require `sidecar` topology. It mounts the credential only into the dedicated network supervisor container. OpenShell rejects credential Secrets with `combined` topology because Kubernetes `fsGroup` volume permission handling can make a shared credential mount readable by the sandbox group. + ## RBAC The chart creates the following RBAC resources in the release namespace: diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index 2cd10b8a0b..96d21243bd 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -428,6 +428,32 @@ supervisor_sideload_method = "image-volume" # filesystem, and network enforcement in the agent container. "sidecar" moves # pod-level network enforcement and gateway session handling into a network sidecar. topology = "combined" +# Optional corporate HTTP forward proxy for policy-approved TLS egress. The +# sandbox workload cannot select or override these settings. Only http:// proxy +# endpoints and TLS CONNECT traffic are supported; plain HTTP egress remains +# direct. `no_proxy` bypasses only the corporate proxy, never OpenShell policy. +# https_proxy = "http://proxy.corp.example:8080" +# no_proxy = ".svc,.svc.cluster.local,10.96.0.0/12,10.244.0.0/16" +# Proxy credentials must be an existing Secret in the sandbox namespace. The +# key contains a `user:pass` value and is mounted only in the network +# supervisor container, never in workload environment or command arguments. +# proxy_auth_secret_name = "corporate-proxy-auth" +# proxy_auth_secret_key = "credentials" +# The gateway validates the Secret name/key syntax and their configuration +# relationship at startup; it does not read the Secret from the Kubernetes API. +# Kubernetes resolves the Secret when the Sandbox Pod starts. A missing key or +# Secret prevents that Pod from starting; unreadable or malformed `user:pass` +# content is validated fail-closed by the supervisor at startup and never +# falls back to direct egress. +# Proxy credential Secrets require `topology = "sidecar"`. Combined topology +# shares its credential mount with the workload and can make it readable by the +# sandbox group through Kubernetes `fsGroup` volume permission handling. +# Required with a credential Secret: Basic authentication to an http:// proxy +# is cleartext on the connection to that proxy. +# proxy_auth_allow_insecure = true +# Last resort for hostname-filtering proxy ACLs. The proxy resolves the target, +# so its ACL becomes part of the egress boundary for proxied connections. +# proxy_connect_by_hostname = true grpc_endpoint = "https://openshell-gateway.agents.svc:8080" ssh_socket_path = "/run/openshell/ssh.sock" client_tls_secret_name = "openshell-client-tls" diff --git a/docs/reference/sandbox-compute-drivers.mdx b/docs/reference/sandbox-compute-drivers.mdx index 2132f3360e..9f2f0ca224 100644 --- a/docs/reference/sandbox-compute-drivers.mdx +++ b/docs/reference/sandbox-compute-drivers.mdx @@ -322,6 +322,12 @@ For maintainer-level implementation details, refer to the [Kubernetes driver REA | `supervisor_image_pull_policy` | `supervisor.image.pullPolicy` | Set the Kubernetes image pull policy for the supervisor image. | | `supervisor_sideload_method` | `supervisor.sideloadMethod` | How the supervisor binary is delivered into sandbox pods. Leave empty to auto-detect from cluster version. Set to `image-volume` to mount the supervisor OCI image directly as a volume (requires Kubernetes 1.33+ with the ImageVolume feature gate; GA in 1.36), or `init-container` to copy it through an init container on older clusters. | | `topology` | `supervisor.topology` | Set `combined` for the default single supervisor path, or `sidecar` to move pod-level network enforcement and the gateway session into a dedicated sidecar. | +| `https_proxy` | `upstreamProxy.url` | Set the operator-owned `http://host:port` corporate forward proxy used for policy-approved TLS CONNECT egress. | +| `no_proxy` | `upstreamProxy.noProxy` | Set destinations that bypass only the corporate proxy. OpenShell policy evaluation still applies. | +| `proxy_auth_secret_name` | `upstreamProxy.authSecret.name` | Set the existing Secret name in the sandbox namespace that contains the proxy credential. Requires `sidecar` topology. | +| `proxy_auth_secret_key` | `upstreamProxy.authSecret.key` | Set the Secret key containing the `user:pass` credential. Requires `sidecar` topology. | +| `proxy_auth_allow_insecure` | `upstreamProxy.authAllowInsecure` | Set `true` to acknowledge that Basic authentication to an HTTP proxy is cleartext. Required with a proxy credential Secret. | +| `proxy_connect_by_hostname` | `upstreamProxy.connectByHostname` | Send hostnames rather than validated IPs in CONNECT requests. Use only when proxy ACLs require hostname targets. | | `sidecar.proxy_uid` | `supervisor.sidecar.proxyUid` | Non-root UID used by the relaxed sidecar when process/binary-aware network policy is disabled. The default binary-aware sidecar runs as UID 0. The network init container exempts the effective sidecar UID from proxy redirection. | | `sidecar.process_binary_aware_network_policy` | `supervisor.sidecar.processBinaryAwareNetworkPolicy` | Keep process/binary-aware network policy enabled in `sidecar` topology. The default runs the sidecar as UID 0 with `SYS_PTRACE` and `DAC_READ_SEARCH`. Set false to run as `proxy_uid`, drop both capabilities, and enforce endpoint/L7 policy without matching `policy.binaries`. | | `app_armor_profile` | `server.appArmorProfile` | Set the sandbox agent container's AppArmor profile. Helm defaults this to `Unconfined` so AppArmor-enabled nodes do not block supervisor network namespace setup. Set the Helm value to an empty string to omit the field, or use `RuntimeDefault` or `Localhost/` for operator-managed profiles. | diff --git a/e2e/rust/Cargo.toml b/e2e/rust/Cargo.toml index 3353f07af7..ca50846ec6 100644 --- a/e2e/rust/Cargo.toml +++ b/e2e/rust/Cargo.toml @@ -89,6 +89,11 @@ name = "readyz_health" path = "tests/readyz_health.rs" required-features = ["e2e-kubernetes"] +[[test]] +name = "kubernetes_corporate_proxy" +path = "tests/kubernetes_corporate_proxy.rs" +required-features = ["e2e-kubernetes"] + [[test]] name = "credential_drivers" path = "tests/credential_drivers.rs" diff --git a/e2e/rust/src/harness/container.rs b/e2e/rust/src/harness/container.rs index 764ee6d0f6..eb24aac1e7 100644 --- a/e2e/rust/src/harness/container.rs +++ b/e2e/rust/src/harness/container.rs @@ -197,6 +197,115 @@ pub struct SupportContainer { engine: ContainerEngine, } +/// A TCP fixture published on the test host for Kubernetes sandbox e2e tests. +/// +/// Kubernetes sandboxes reach it through the chart-provided +/// `host.openshell.internal` alias. Unlike [`SupportContainer`], this does not +/// require the Docker e2e network used by local-container driver tests. +pub struct HostSupportContainer { + pub port: u16, + container_id: String, + engine: ContainerEngine, +} + +impl HostSupportContainer { + /// Start a Python fixture and publish `container_port` on a free host port. + pub async fn start_python(script: &str, container_port: u16) -> Result { + Self::start_python_on_host_port(script, container_port, find_free_port()).await + } + + /// Start a Python fixture on a caller-selected host port. + /// + /// Use this when the fixture endpoint must be known before the Helm chart + /// starts the gateway, such as the configured corporate forward proxy. + pub async fn start_python_on_host_port( + script: &str, + container_port: u16, + port: u16, + ) -> Result { + let engine = ContainerEngine::from_env()?; + let output = engine + .command() + .args([ + "run", + "--detach", + "--entrypoint", + "python3", + "-p", + &format!("{port}:{container_port}"), + DEFAULT_TEST_SERVER_IMAGE, + "-c", + script, + ]) + .output() + .map_err(|err| format!("start {} host fixture: {err}", engine.name()))?; + if !output.status.success() { + return Err(format!( + "{} run failed (exit {:?}):\n{}", + engine.name(), + output.status.code(), + String::from_utf8_lossy(&output.stderr) + )); + } + let fixture = Self { + port, + container_id: String::from_utf8_lossy(&output.stdout).trim().to_string(), + engine, + }; + fixture.wait_until_listening(container_port).await?; + Ok(fixture) + } + + async fn wait_until_listening(&self, container_port: u16) -> Result<(), String> { + let deadline = timeout(Duration::from_secs(60), async { + let mut tick = interval(Duration::from_millis(500)); + loop { + tick.tick().await; + let output = self + .engine + .command() + .args(["exec", &self.container_id, "python3", "-c", &format!("import socket; socket.create_connection(('127.0.0.1', {container_port}), timeout=1).close()")]) + .output() + .ok(); + if output.is_some_and(|output| output.status.success()) { + return; + } + } + }) + .await; + deadline.map_err(|_| { + format!( + "host fixture did not listen within 60s. Logs:\n{}", + self.logs().unwrap_or_else(|err| err) + ) + }) + } + + pub fn logs(&self) -> Result { + let output = self + .engine + .command() + .args(["logs", &self.container_id]) + .output() + .map_err(|err| format!("read {} fixture logs: {err}", self.engine.name()))?; + Ok(format!( + "{}{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + )) + } +} + +impl Drop for HostSupportContainer { + fn drop(&mut self) { + let _ = self + .engine + .command() + .args(["rm", "-f", &self.container_id]) + .output(); + } +} + impl SupportContainer { /// Start a `python3 -c