Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 24 additions & 0 deletions .github/workflows/branch-e2e.yml
Original file line number Diff line number Diff line change
Expand Up @@ -278,12 +278,22 @@ jobs:
vm-e2e:
needs: [pr_metadata, build-cli, build-conformance, build-gateway, build-vm-driver]
if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true'
strategy:
fail-fast: false
matrix:
include:
- suite: managed
test_name: ""
- suite: additional-ca
test_name: additional_ca
permissions:
actions: read
contents: read
packages: read
uses: ./.github/workflows/e2e-vm-test.yml
with:
suite-name: ${{ matrix.suite }}
test-name: ${{ matrix.test_name }}
conformance-artifact-prefix: openshell-conformance

docker-external-driver-e2e:
Expand Down Expand Up @@ -356,14 +366,27 @@ jobs:
agent_sandbox_version: v0.5.0
topology: combined
extra_helm_values: ""
test_name: ""
- agent_sandbox_api: v1alpha1
agent_sandbox_version: v0.4.6
topology: combined
extra_helm_values: ""
test_name: ""
- agent_sandbox_api: v1beta1
agent_sandbox_version: v0.5.0
topology: sidecar
extra_helm_values: deploy/helm/openshell/ci/values-sidecar.yaml
test_name: ""
- agent_sandbox_api: v1beta1
agent_sandbox_version: v0.5.0
topology: combined-additional-ca
extra_helm_values: ""
test_name: additional_ca
- agent_sandbox_api: v1beta1
agent_sandbox_version: v0.5.0
topology: sidecar-additional-ca
extra_helm_values: deploy/helm/openshell/ci/values-sidecar.yaml
test_name: additional_ca
permissions:
actions: read
contents: read
Expand All @@ -374,6 +397,7 @@ jobs:
job-name: Kubernetes E2E (Rust smoke, ${{ matrix.topology }}, Agent Sandbox ${{ matrix.agent_sandbox_api }})
agent-sandbox-version: ${{ matrix.agent_sandbox_version }}
extra-helm-values: ${{ matrix.extra_helm_values }}
test-name: ${{ matrix.test_name }}
conformance-artifact-prefix: openshell-conformance

kubernetes-workspace-managed-e2e:
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/e2e-docker-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ on:
required: false
type: string
default: >-
[{"suite":"python","cmd":"mise run --no-deps --skip-deps e2e:python","apt_packages":"","python_proto":true,"mcp":false},{"suite":"oidc-python","cmd":"mise run --no-deps --skip-deps e2e:oidc-python:docker","apt_packages":"","python_proto":true,"mcp":false},{"suite":"oidc-pkce-docker","cmd":"mise run --no-deps --skip-deps e2e:oidc-pkce:docker","apt_packages":"openssh-client","python_proto":false,"mcp":false},{"suite":"rust-docker","cmd":"mise run --no-deps --skip-deps e2e:rust","apt_packages":"openssh-client","python_proto":false,"mcp":false},{"suite":"mcp","cmd":"mise run --no-deps --skip-deps e2e:mcp","apt_packages":"","python_proto":false,"mcp":true}]
[{"suite":"python","cmd":"mise run --no-deps --skip-deps e2e:python","apt_packages":"","python_proto":true,"mcp":false},{"suite":"oidc-python","cmd":"mise run --no-deps --skip-deps e2e:oidc-python:docker","apt_packages":"","python_proto":true,"mcp":false},{"suite":"oidc-pkce-docker","cmd":"mise run --no-deps --skip-deps e2e:oidc-pkce:docker","apt_packages":"openssh-client","python_proto":false,"mcp":false},{"suite":"rust-docker","cmd":"mise run --no-deps --skip-deps e2e:rust","apt_packages":"openssh-client","python_proto":false,"mcp":false},{"suite":"additional-ca","cmd":"OPENSHELL_E2E_DOCKER_TEST=additional_ca mise run --no-deps --skip-deps e2e:docker","apt_packages":"openssh-client","python_proto":false,"mcp":false},{"suite":"mcp","cmd":"mise run --no-deps --skip-deps e2e:mcp","apt_packages":"","python_proto":false,"mcp":true}]

permissions:
actions: read
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/e2e-podman-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ on:
required: false
type: string
default: >-
[{"suite":"provider-refresh-keycloak","runner":"ubuntu-26.04","podman_major":"5","podman_package_version":"5.7.0+ds2-3build1","conmon_package_version":"2.1.13+ds1-2","cmd":"mise run --no-deps --skip-deps e2e:provider-refresh-keycloak"}]
[{"suite":"provider-refresh-keycloak","runner":"ubuntu-26.04","podman_major":"5","podman_package_version":"5.7.0+ds2-3build1","conmon_package_version":"2.1.13+ds1-2","cmd":"mise run --no-deps --skip-deps e2e:provider-refresh-keycloak"},{"suite":"additional-ca","runner":"ubuntu-26.04","podman_major":"5","podman_package_version":"5.7.0+ds2-3build1","conmon_package_version":"2.1.13+ds1-2","cmd":"OPENSHELL_E2E_PODMAN_TEST=additional_ca mise run --no-deps --skip-deps e2e:podman"}]

permissions:
actions: read
Expand Down
5 changes: 5 additions & 0 deletions .github/workflows/e2e-vm-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,10 @@ on:
required: false
type: string
default: ""
test-name:
required: false
type: string
default: ""

permissions:
actions: read
Expand Down Expand Up @@ -61,4 +65,5 @@ jobs:
- name: Run tests
env:
E2E_TASK: ${{ inputs.e2e-task }}
OPENSHELL_E2E_VM_TEST: ${{ inputs.test-name }}
run: mise run --no-deps --skip-deps "$E2E_TASK"
4 changes: 4 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

24 changes: 24 additions & 0 deletions architecture/sandbox.md
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,30 @@ maps an accepted proxy connection back to the workload socket by matching the
complete local-to-remote TCP tuple before resolving every process that owns the
socket inode.

### Additional Destination Trust

`[openshell.supervisor.network].additional_ca_cert_paths` adds operator-owned
certificate authorities to sandbox destination TLS trust. Gateway startup reads
and strictly normalizes the certificate-only PEM files into one bundle. An
invalid source or staging failure aborts startup or sandbox provisioning rather
than falling back to public roots. The bundle augments bundled, system, or
native roots and preserves normal hostname verification.

Drivers deliver the normalized material at their runtime boundary. Docker and
Podman bind-mount a gateway-owned artifact. Kubernetes server-side-applies one
gateway-scoped ConfigMap per target namespace; combined topology mounts it only
in the agent container that runs the supervisor, while sidecar topology mounts
it only in `openshell-network`. VM writes it into the per-sandbox overlay. All
paths converge on `/etc/openshell-tls/network-additional-ca.crt` and the
operator-controlled `--network-additional-ca-bundle` argument. The supervisor
also includes these roots in its child-process trust files.

Destination trust remains separate from corporate-proxy CA material, gateway
listener TLS, OIDC trust, and sandbox-to-gateway mTLS through
`OPENSHELL_TLS_CA`. Configuration is startup-only. Source changes require a
gateway restart, and running sandboxes retain their initialized trust until
recreated or restarted.

CONNECT and absolute-form forward HTTP are explicit-proxy adapters over the same
egress pipeline. Each adapter normalizes its request into an egress intent, and
the shared authorization result carries the process evidence and endpoint
Expand Down
4 changes: 3 additions & 1 deletion crates/openshell-core/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -54,8 +54,10 @@ tonic-prost-build = { workspace = true }
protoc-bin-vendored = { workspace = true }

[dev-dependencies]
tempfile = "3"
h2 = "0.4"
rcgen = { workspace = true }
tempfile = "3"
tokio-rustls = { workspace = true }

[lints]
workspace = true
5 changes: 5 additions & 0 deletions crates/openshell-core/src/container_paths.rs
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,10 @@ pub const CONTROL_ROOTS: &[&str] = &[
pub const SUPERVISOR_CONTAINER_DIR: &str = "/opt/openshell/bin";
pub const SUPERVISOR_CONTAINER_BINARY: &str = "/opt/openshell/bin/openshell-sandbox";
pub const TLS_CLIENT_DIR: &str = "/etc/openshell/tls/client";
/// Destination-only trust staged by compute drivers for the network
/// supervisor. This must remain separate from gateway client mTLS and the
/// corporate-proxy CA path.
pub const NETWORK_ADDITIONAL_CA_BUNDLE_PATH: &str = "/etc/openshell-tls/network-additional-ca.crt";
pub const TLS_CA_MOUNT_PATH: &str = "/etc/openshell/tls/client/ca.crt";
pub const TLS_CERT_MOUNT_PATH: &str = "/etc/openshell/tls/client/tls.crt";
pub const TLS_KEY_MOUNT_PATH: &str = "/etc/openshell/tls/client/tls.key";
Expand Down Expand Up @@ -108,6 +112,7 @@ mod tests {
SUPERVISOR_CONTAINER_DIR,
SUPERVISOR_CONTAINER_BINARY,
TLS_CLIENT_DIR,
NETWORK_ADDITIONAL_CA_BUNDLE_PATH,
TLS_CA_MOUNT_PATH,
TLS_CERT_MOUNT_PATH,
TLS_KEY_MOUNT_PATH,
Expand Down
185 changes: 161 additions & 24 deletions crates/openshell-core/src/grpc_client.rs
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,26 @@ impl tonic::service::Interceptor for AuthInterceptor {
///
/// When the endpoint uses `http://`, a plaintext connection is used (for
/// deployments where TLS is disabled, e.g. behind a Cloudflare Tunnel).
fn configure_gateway_tls(
ep: Endpoint,
ca_pem: Vec<u8>,
cert_pem: Vec<u8>,
key_pem: Vec<u8>,
server_name: Option<&str>,
) -> Result<Endpoint> {
// Trust only the configured gateway CA. Destination, system, and proxy
// trust are intentionally not parameters to this control-plane boundary.
let mut tls_config = ClientTlsConfig::new()
.ca_certificate(Certificate::from_pem(ca_pem))
.identity(Identity::from_pem(cert_pem, key_pem));
if let Some(server_name) = server_name.filter(|name| !name.is_empty()) {
tls_config = tls_config.domain_name(server_name);
}
ep.tls_config(tls_config)
.into_diagnostic()
.wrap_err("failed to configure TLS")
}

async fn build_plain_channel(endpoint: &str) -> Result<Channel> {
let mut ep = Endpoint::from_shared(endpoint.to_string())
.into_diagnostic()
Expand Down Expand Up @@ -172,30 +192,12 @@ async fn build_plain_channel(endpoint: &str) -> Result<Channel> {
.into_diagnostic()
.wrap_err_with(|| format!("failed to read client key from {key_path}"))?;

// Trust only the configured CA — this is the chart's internal CA
// that signs both the gateway's internal server certificate and
// this client's identity certificate. The gateway uses SNI-based
// certificate selection to present this internal cert to supervisor
// connections, so no public root trust is needed here.
//
// Do NOT add `.with_native_roots()` or `.with_webpki_roots()` here:
// the supervisor runs inside the user-selected sandbox image
// (Docker/Podman drivers), and broadening the trust store would let
// an attacker who controls the image + DNS present a publicly valid
// certificate and intercept the supervisor→gateway TLS connection.
let mut tls_config = ClientTlsConfig::new()
.ca_certificate(Certificate::from_pem(ca_pem))
.identity(Identity::from_pem(cert_pem, key_pem));
if let Ok(server_name) = std::env::var(sandbox_env::GATEWAY_TLS_SERVER_NAME)
&& !server_name.is_empty()
{
tls_config = tls_config.domain_name(server_name);
}

ep = ep
.tls_config(tls_config)
.into_diagnostic()
.wrap_err("failed to configure TLS")?;
// The gateway uses SNI-based certificate selection to present its
// internal cert to supervisor connections. Do not add native,
// webpki, destination, or child-process roots here: an image that can
// influence DNS must not broaden supervisor→gateway authentication.
let server_name = std::env::var(sandbox_env::GATEWAY_TLS_SERVER_NAME).ok();
ep = configure_gateway_tls(ep, ca_pem, cert_pem, key_pem, server_name.as_deref())?;
}

ep.connect()
Expand All @@ -204,6 +206,141 @@ async fn build_plain_channel(endpoint: &str) -> Result<Channel> {
.wrap_err("failed to connect to OpenShell server")
}

#[cfg(test)]
mod gateway_tls_isolation_tests {
use super::*;
use rcgen::{CertificateParams, IsCa, KeyPair};
use rustls::pki_types::{CertificateDer, PrivateKeyDer};
use std::io::Cursor;
use tokio::net::TcpListener;
use tokio_rustls::TlsAcceptor;

#[allow(clippy::struct_field_names)]
struct TestPki {
ca_pem: Vec<u8>,
server_cert_pem: Vec<u8>,
server_key_pem: Vec<u8>,
client_cert_pem: Vec<u8>,
client_key_pem: Vec<u8>,
}

fn test_pki(server_name: &str) -> TestPki {
let mut ca_params = CertificateParams::new(Vec::<String>::new()).unwrap();
ca_params.is_ca = IsCa::Ca(rcgen::BasicConstraints::Unconstrained);
ca_params.key_usages = vec![
rcgen::KeyUsagePurpose::KeyCertSign,
rcgen::KeyUsagePurpose::CrlSign,
];
let ca_key = KeyPair::generate().unwrap();
let ca = ca_params.self_signed(&ca_key).unwrap();

let server_key = KeyPair::generate().unwrap();
let server = CertificateParams::new(vec![server_name.to_string()])
.unwrap()
.signed_by(&server_key, &ca, &ca_key)
.unwrap();
let client_key = KeyPair::generate().unwrap();
let client = CertificateParams::new(Vec::<String>::new())
.unwrap()
.signed_by(&client_key, &ca, &ca_key)
.unwrap();
TestPki {
ca_pem: ca.pem().into_bytes(),
server_cert_pem: server.pem().into_bytes(),
server_key_pem: server_key.serialize_pem().into_bytes(),
client_cert_pem: client.pem().into_bytes(),
client_key_pem: client_key.serialize_pem().into_bytes(),
}
}

async fn start_h2_tls_server(pki: &TestPki) -> std::net::SocketAddr {
let certificates = rustls_pemfile::certs(&mut Cursor::new(&pki.server_cert_pem))
.collect::<std::result::Result<Vec<CertificateDer<'static>>, _>>()
.unwrap();
let key: PrivateKeyDer<'static> =
rustls_pemfile::private_key(&mut Cursor::new(&pki.server_key_pem))
.unwrap()
.unwrap();
let mut config = rustls::ServerConfig::builder()
.with_no_client_auth()
.with_single_cert(certificates, key)
.unwrap();
config.alpn_protocols = vec![b"h2".to_vec()];
let acceptor = TlsAcceptor::from(Arc::new(config));
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
tokio::spawn(async move {
let Ok((stream, _)) = listener.accept().await else {
return;
};
let Ok(stream) = acceptor.accept(stream).await else {
return;
};
if let Ok(mut connection) = h2::server::handshake(stream).await {
while let Some(request) = connection.accept().await {
if request.is_err() {
break;
}
}
}
});
address
}

async fn connect_with_gateway_ca(
address: std::net::SocketAddr,
pki: &TestPki,
server_name: &str,
) -> Result<Channel> {
let endpoint = Endpoint::from_shared(format!("https://{address}")).into_diagnostic()?;
configure_gateway_tls(
endpoint,
pki.ca_pem.clone(),
pki.client_cert_pem.clone(),
pki.client_key_pem.clone(),
Some(server_name),
)?
.connect()
.await
.into_diagnostic()
}

#[tokio::test]
async fn destination_ca_cannot_authenticate_gateway() {
const GATEWAY_NAME: &str = "gateway.internal.test";
let gateway = test_pki(GATEWAY_NAME);
let destination = test_pki(GATEWAY_NAME);

let real_address = start_h2_tls_server(&gateway).await;
tokio::time::timeout(
Duration::from_secs(5),
connect_with_gateway_ca(real_address, &gateway, GATEWAY_NAME),
)
.await
.expect("real gateway connection timed out")
.expect("configured gateway CA should authenticate the gateway");

let fake_address = start_h2_tls_server(&destination).await;
let error = tokio::time::timeout(
Duration::from_secs(5),
connect_with_gateway_ca(fake_address, &gateway, GATEWAY_NAME),
)
.await
.expect("fake gateway rejection timed out")
.expect_err("destination-signed fake gateway must be rejected");
assert!(format!("{error:?}").contains("certificate"), "{error:?}");

let mismatch_address = start_h2_tls_server(&gateway).await;
tokio::time::timeout(
Duration::from_secs(5),
connect_with_gateway_ca(mismatch_address, &gateway, "wrong.internal.test"),
)
.await
.expect("hostname mismatch rejection timed out")
.expect_err("normal server-name verification must remain enabled");
}
}

/// Build a Bearer-authenticated channel to the gateway.
///
/// First call per process resolves the sandbox JWT via the three-step
Expand Down
Loading
Loading