diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5b69bce63..196b838a5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -62,6 +62,12 @@ dstack deploy ./docker-compose.yml \ --no-tee ``` +To exercise persistent TPM-backed app keys as well, install `swtpm` on the VMM +host and select `key_provider=tpm` in the VMM console (or pass `--key-provider +tpm` in tooling that exposes the compose option). The VMM keeps the software +TPM state in the VM work directory so that seal/unseal survives guest restarts. +The software TPM is host-controlled and does not provide hardware isolation. + The simulator package is installed only in development images. This mode provides no hardware isolation and must never be used with production workloads or secrets. Real quote generation, hardware isolation, and KMS diff --git a/docs/security/cvm-boundaries.md b/docs/security/cvm-boundaries.md index e35496727..fed4f084f 100644 --- a/docs/security/cvm-boundaries.md +++ b/docs/security/cvm-boundaries.md @@ -44,7 +44,7 @@ This is the main configuration file for the application in JSON format: | init_script | 0.5.5 | string | Bash script that executed prior to dockerd startup | | storage_fs | 0.5.5 | string | Filesystem type for the data disk of the CVM. Supported values: "zfs", "ext4". default to "zfs". **ZFS:** Ensures filesystem integrity with built-in data protection features. **ext4:** Provides better performance for database applications with lower overhead and faster I/O operations, but no strong integrity protection. | | swap_size | 0.5.5 | string/integer | The linux swap size. default to 0. Can be in byte or human-readable format (e.g., "1G", "256M"). | -| key_provider | 0.5.6 | string | Key provider type. Supported values: "none", "kms", "local", "tpm". `"tpm"` is only supported on platforms whose TPM is part of the platform trust model (GCP vTPM, AWS EC2 NitroTPM); on other platforms guest setup fails closed, since sealing to a host-provided software TPM (e.g. swtpm under bare QEMU) offers no protection against the host. | +| key_provider | 0.5.6 | string | Key provider type. Supported values: "none", "kms", "local", "tpm". GCP vTPM and AWS EC2 NitroTPM are part of their platform trust models. The Dstack platform can use VMM-managed swtpm for seal/unseal and restart persistence, but it offers no protection against the host and is intentionally not accepted by remote verifiers. | The hash of this file content is extended as the dstack `compose-hash` launch event. On TDX-family platforms the launch event is measured into RTMR3. On AWS NitroTPM it is measured into non-resettable SHA384 PCR14 before the `system-ready` launch boundary. Remote verifiers extract and replay this event during attestation. diff --git a/dstack/Cargo.lock b/dstack/Cargo.lock index ee581e254..f785ebacc 100644 --- a/dstack/Cargo.lock +++ b/dstack/Cargo.lock @@ -2135,6 +2135,7 @@ dependencies = [ "humantime", "insta", "key-provider-client", + "libc", "load_config", "lspci", "nix 0.29.0", @@ -2157,6 +2158,7 @@ dependencies = [ "supervisor-client", "tailf", "tar", + "tempfile", "tokio", "tracing", "tracing-subscriber", diff --git a/dstack/tpm-attest/src/lib.rs b/dstack/tpm-attest/src/lib.rs index 7105af79d..a0f3046a2 100644 --- a/dstack/tpm-attest/src/lib.rs +++ b/dstack/tpm-attest/src/lib.rs @@ -48,7 +48,7 @@ const AWS_NITRO_PCRS: [u32; 5] = [4, 7, 8, 12, 14]; pub fn dstack_pcr_policy_for_platform(platform: Platform) -> Result { match platform { - Platform::Gcp => Ok(dstack_pcr_policy()), + Platform::Dstack | Platform::Gcp => Ok(dstack_pcr_policy()), Platform::AwsEc2 => Ok(PcrSelection::sha384(&AWS_NITRO_PCRS)), _ => bail!("TPM local key provider is not supported on {platform:?}"), } @@ -367,6 +367,16 @@ mod tests { let policy = dstack_pcr_policy_for_platform(Platform::AwsEc2).unwrap(); assert_eq!(policy.to_arg(), "sha384:4,7,8,12,14"); } + + #[test] + fn dstack_platform_uses_development_tpm_policy() { + assert_eq!( + dstack_pcr_policy_for_platform(Platform::Dstack) + .unwrap() + .to_arg(), + "sha256:0,2,14" + ); + } } mod gcp_ak; diff --git a/dstack/vmm/Cargo.toml b/dstack/vmm/Cargo.toml index ccc2a2be8..6d5203776 100644 --- a/dstack/vmm/Cargo.toml +++ b/dstack/vmm/Cargo.toml @@ -15,6 +15,7 @@ rocket-vsock-listener = { workspace = true } tracing.workspace = true tracing-subscriber = { workspace = true, features = ["env-filter"] } anyhow.workspace = true +libc.workspace = true serde = { workspace = true, features = ["derive"] } serde_json.workspace = true shared_child.workspace = true @@ -63,6 +64,7 @@ tar.workspace = true [dev-dependencies] insta.workspace = true +tempfile.workspace = true [build-dependencies] or-panic.workspace = true diff --git a/dstack/vmm/src/app.rs b/dstack/vmm/src/app.rs index ce51dfc09..e303e58e7 100644 --- a/dstack/vmm/src/app.rs +++ b/dstack/vmm/src/app.rs @@ -47,6 +47,30 @@ pub(crate) mod registry; mod vm_info; mod workdir; +fn signal_pidfd(pid: u32, signal: libc::c_int) -> std::io::Result<()> { + // SAFETY: pidfd syscalls receive scalar arguments and a null siginfo pointer. + let fd = unsafe { libc::syscall(libc::SYS_pidfd_open, pid, 0) }; + if fd < 0 { + return Err(std::io::Error::last_os_error()); + } + let result = unsafe { + libc::syscall( + libc::SYS_pidfd_send_signal, + fd, + signal, + std::ptr::null::(), + 0, + ) + }; + // SAFETY: fd was returned by pidfd_open and is owned by this function. + unsafe { libc::close(fd as libc::c_int) }; + if result < 0 { + Err(std::io::Error::last_os_error()) + } else { + Ok(()) + } +} + #[derive(Deserialize, Serialize, Debug, Clone)] pub struct PortMapping { pub address: IpAddr, @@ -425,10 +449,44 @@ impl App { pub async fn stop_vm(&self, id: &str) -> Result<()> { self.set_started(id, false)?; - self.supervisor.stop(id).await?; + self.stop_vm_process(id).await?; Ok(()) } + async fn stop_vm_process(&self, id: &str) -> Result<()> { + let Some(info) = self.supervisor.info(id).await? else { + return Ok(()); + }; + // Non-TPM VMs run QEMU directly and keep the existing Supervisor stop + // path. Only the TPM launcher's hidden subcommand implements graceful + // child-process shutdown. + if info.config.args.first().map(String::as_str) != Some("vm-launcher") { + return self.supervisor.stop(id).await; + } + if info.state.status.is_running() { + let pid = info.state.pid.context("running VM launcher has no PID")?; + if let Err(error) = signal_pidfd(pid, libc::SIGTERM) { + warn!(id, %pid, %error, "failed to signal VM launcher gracefully; forcing shutdown"); + return self.supervisor.stop(id).await; + } + for _ in 0..150 { + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + let running = self + .supervisor + .info(id) + .await? + .is_some_and(|info| info.state.status.is_running()); + if !running { + // Synchronize Supervisor's `started` flag after the launcher + // completed its graceful child cleanup. + return self.supervisor.stop(id).await; + } + } + warn!(id, "VM launcher did not stop gracefully; forcing shutdown"); + } + self.supervisor.stop(id).await + } + pub async fn remove_vm(&self, id: &str) -> Result<()> { { let mut state = self.lock(); @@ -464,8 +522,8 @@ impl App { /// `delete_workdir`: true for user-initiated removal, false for orphan cleanup. async fn finish_remove_vm(&self, id: &str, delete_workdir: bool) -> Result<()> { // Stop the supervisor process (idempotent if already stopped) - if let Err(err) = self.supervisor.stop(id).await { - debug!("supervisor.stop({id}) during removal: {err:?}"); + if let Err(err) = self.stop_vm_process(id).await { + debug!("graceful VM stop during removal failed: {err:?}"); } // Poll until the process is no longer running, then remove it. diff --git a/dstack/vmm/src/app/qemu.rs b/dstack/vmm/src/app/qemu.rs index 6850bb653..270c62c73 100644 --- a/dstack/vmm/src/app/qemu.rs +++ b/dstack/vmm/src/app/qemu.rs @@ -6,6 +6,7 @@ use crate::{ app::Manifest, config::{CvmConfig, Networking, NetworkingMode, ProcessAnnotation, TeePlatform}, + vm_launcher::{ChildCommand, LaunchSpec}, }; use std::collections::HashMap; use std::os::unix::fs::PermissionsExt; @@ -29,6 +30,7 @@ use anyhow::{bail, Context, Result}; use bon::Builder; use dstack_types::{shared_filenames::HOST_SHARED_DISK_LABEL, KeyProviderKind}; use fs_err as fs; +use nix::unistd::User; use serde::Serialize; use supervisor_client::supervisor::ProcessConfig; @@ -179,7 +181,8 @@ struct PreparedQemuLaunch { hugepage_numa_nodes: Option>, gpu_numa_nodes: HashMap, numa_cpus: Option, - tpm_path: Option<&'static str>, + swtpm_socket: Option, + swtpm_path: Option, tdx_mr_config_id: Option, snp_host_data: Option, snp_launch_params: Option, @@ -219,11 +222,20 @@ impl PreparedQemuLaunch { } else { None }; - let tpm_path = if matches!(app_compose.key_provider(), KeyProviderKind::Tpm) { - Some(detect_tpm_device()?) - } else { - None - }; + let (swtpm_socket, swtpm_path) = + if matches!(app_compose.key_provider(), KeyProviderKind::Tpm) { + let swtpm_path = which::which("swtpm") + .context("tpm key provider requested but swtpm is not installed")?; + let state_dir = workdir.swtpm_state_dir(); + fs::create_dir_all(&state_dir).context("failed to create swtpm state directory")?; + let socket = workdir.swtpm_socket(); + if socket.exists() { + fs::remove_file(&socket).context("failed to remove stale swtpm socket")?; + } + (Some(socket), Some(swtpm_path)) + } else { + (None, None) + }; prepare_shared_disk(&workdir, cfg)?; let tee_enabled = !vm.manifest.no_tee; @@ -257,7 +269,8 @@ impl PreparedQemuLaunch { hugepage_numa_nodes, gpu_numa_nodes, numa_cpus, - tpm_path, + swtpm_socket, + swtpm_path, tdx_mr_config_id, snp_host_data, snp_launch_params, @@ -301,16 +314,6 @@ fn prepare_shared_disk(workdir: &VmWorkDir, cfg: &CvmConfig) -> Result<()> { create_shared_disk(&shared_disk_path, shared_dir).context("failed to create shared disk") } -fn detect_tpm_device() -> Result<&'static str> { - if Path::new("/dev/tpmrm0").exists() { - Ok("/dev/tpmrm0") - } else if Path::new("/dev/tpm0").exists() { - Ok("/dev/tpm0") - } else { - bail!("tpm key provider requested but no TPM device found on host") - } -} - struct QemuCommandBuilder<'a> { vm: &'a VmConfig, cfg: &'a CvmConfig, @@ -333,7 +336,71 @@ impl VmConfig { prepared: &prepared, } .build()?; - Ok(vec![process]) + let Some(socket) = prepared.swtpm_socket.as_deref() else { + return Ok(vec![process]); + }; + let swtpm_path = prepared + .swtpm_path + .as_ref() + .context("missing swtpm executable for configured socket")?; + let (socket_uid, socket_gid) = if cfg.user.is_empty() { + (unsafe { libc::geteuid() }, unsafe { libc::getegid() }) + } else { + let user = User::from_name(&cfg.user) + .context("failed to resolve QEMU user")? + .with_context(|| format!("QEMU user {} does not exist", cfg.user))?; + (user.uid.as_raw(), user.gid.as_raw()) + }; + + let swtpm_args = vec![ + "socket".into(), + "--tpm2".into(), + "--tpmstate".into(), + format!("dir={}", prepared.workdir.swtpm_state_dir().display()), + "--ctrl".into(), + format!( + "type=unixio,path={},mode=0600,uid={socket_uid},gid={socket_gid}", + socket.display() + ), + "--flags".into(), + "not-need-init,startup-clear".into(), + ]; + let spec = LaunchSpec { + qemu: ChildCommand { + command: process.command, + args: process.args, + }, + swtpm: ChildCommand { + command: swtpm_path.to_string_lossy().into_owned(), + args: swtpm_args, + }, + swtpm_socket: socket.to_path_buf(), + startup_timeout_ms: 5_000, + shutdown_timeout_ms: 10_000, + }; + let spec_path = prepared.workdir.launch_spec_path(); + safe_write::safe_write(&spec_path, serde_json::to_vec_pretty(&spec)?) + .context("failed to write VM launch specification")?; + let executable = + std::env::current_exe().context("failed to locate dstack-vmm executable")?; + let launcher = ProcessConfig { + id: self.manifest.id.clone(), + name: self.manifest.name.clone(), + command: executable.to_string_lossy().into_owned(), + args: vec![ + "vm-launcher".into(), + "--spec".into(), + spec_path.to_string_lossy().into_owned(), + ], + env: process.env, + cwd: process.cwd, + stdout: process.stdout, + stderr: process.stderr, + pidfile: process.pidfile, + cid: process.cid, + note: process.note, + }; + Ok(vec![launcher]) } } @@ -518,10 +585,12 @@ impl QemuCommandBuilder<'_> { } fn configure_tpm_and_vsock(&self, command: &mut Command) { - if let Some(tpm_path) = self.prepared.tpm_path { + if let Some(socket) = &self.prepared.swtpm_socket { command + .arg("-chardev") + .arg(format!("socket,id=chrtpm,path={}", socket.display())) .arg("-tpmdev") - .arg(format!("passthrough,id=tpm0,path={tpm_path}")) + .arg("emulator,id=tpm0,chardev=chrtpm") .arg("-device") .arg("tpm-tis,tpmdev=tpm0"); } @@ -970,14 +1039,15 @@ mod tests { workdir: PathBuf::from("/does-not-exist/vm-1"), gateway_enabled: false, }; - let prepared = PreparedQemuLaunch { + let mut prepared = PreparedQemuLaunch { workdir: VmWorkDir::new("/does-not-exist/vm-1"), platform: TeePlatform::Tdx, networks: vec![config.cvm.networking.clone(), config.cvm.networking.clone()], hugepage_numa_nodes: None, gpu_numa_nodes: HashMap::new(), numa_cpus: None, - tpm_path: None, + swtpm_socket: None, + swtpm_path: None, tdx_mr_config_id: None, snp_host_data: None, snp_launch_params: None, @@ -1024,5 +1094,25 @@ mod tests { .args .iter() .any(|arg| arg.contains("virtio-net-pci,netdev=net1"))); + + prepared.swtpm_socket = Some(PathBuf::from("/does-not-exist/vm-1/swtpm/swtpm.sock")); + let process = QemuCommandBuilder { + vm: &vm, + cfg: &config.cvm, + gpus: &GpuConfig::default(), + prepared: &prepared, + } + .build() + .unwrap(); + assert!(process.args.windows(2).any(|args| { + args == [ + "-chardev", + "socket,id=chrtpm,path=/does-not-exist/vm-1/swtpm/swtpm.sock", + ] + })); + assert!(process + .args + .windows(2) + .any(|args| args == ["-tpmdev", "emulator,id=tpm0,chardev=chrtpm"])); } } diff --git a/dstack/vmm/src/app/workdir.rs b/dstack/vmm/src/app/workdir.rs index d5fdc338f..c11c99f33 100644 --- a/dstack/vmm/src/app/workdir.rs +++ b/dstack/vmm/src/app/workdir.rs @@ -104,6 +104,18 @@ impl VmWorkDir { self.workdir.join("shared") } + pub fn swtpm_state_dir(&self) -> PathBuf { + self.workdir.join("swtpm") + } + + pub fn swtpm_socket(&self) -> PathBuf { + self.swtpm_state_dir().join("swtpm.sock") + } + + pub fn launch_spec_path(&self) -> PathBuf { + self.workdir.join("launch.json") + } + pub fn app_compose_path(&self) -> PathBuf { self.shared_dir().join(APP_COMPOSE) } diff --git a/dstack/vmm/src/console_v0.html b/dstack/vmm/src/console_v0.html deleted file mode 100644 index d68227724..000000000 --- a/dstack/vmm/src/console_v0.html +++ /dev/null @@ -1,2631 +0,0 @@ - - - - - - - - - {{TITLE}} - - - - - - - - - - - - - - - - - - - - -
- - -
-
-

Deploy a new instance

-
-
-
- - -
- -
- - -
- -
- - -
- -
- -
- - -
-
- -
- -
- - -
- Leave as 0 to disable swap. -
- -
- - -
- -
- - -
- -
- -
-
- - or paste below - -
- -
-
-
- -
- - - -
- -
- - -
- -
- - -
- -
- - -
- -
- -
- -
- -
- - - - - - - - -
-
-
- - -
- -
- -
- -
- -
-
- - -
-
- - -
-
-
-
- Compose Hash: 0x{{ composeHashPreview }} -
- -
- - -
-
-
-
- -

VM List

-
-
-
- - -
-
- Total: {{ totalVMs }} -
-
- - - - - - - - - - - - - -
NameStatusUptimeViewActions
-
- - -
- Page - - of {{ maxPage || 1 }} -
- - - - - -
-
- -
-
-

Update VM Config

- -
-
- - -
-
- -
- - -
-
-
- -
- -
- - -
- Enable "Update compose" to change swap size. -
- -
- - -
- -
-
- - -
- - -
-
- -
- -
- - -
-
-
- -
-
-
- -
-
- - or paste below - -
- -
-
-
- - -
-
- Compose Hash: 0x{{ upgradeComposeHashPreview }} -
-
-
- -
-
- -
-
- -
- -
- -
- - -
- -
- - -
-
-
- -
-
-

Derive VM

-

- This will create a new VM instance with the same app id, but the disk state will NOT migrate to the - new instance. -

- -
- - -
- -
- - -
-
- - -
-
- - -
-
- - -
-
-
- -
-
- -
-
-
- {{ errorMessage }} - -
-
-
- Version: v{{ version.version }} ({{ version.rev }}) -
-
- - - - - - - - - - - - - diff --git a/dstack/vmm/src/main.rs b/dstack/vmm/src/main.rs index 8ab3be383..f2302dc20 100644 --- a/dstack/vmm/src/main.rs +++ b/dstack/vmm/src/main.rs @@ -30,6 +30,7 @@ mod main_routes; mod main_service; mod one_shot; mod openapi; +mod vm_launcher; const CARGO_PKG_VERSION: &str = env!("CARGO_PKG_VERSION"); const GIT_REV: &str = git_version::git_version!( @@ -60,6 +61,9 @@ enum Command { Serve, /// One-shot VM execution mode for debugging Run(RunArgs), + /// Internal per-VM QEMU/swtpm launcher. + #[command(hide = true)] + VmLauncher(VmLauncherArgs), } #[derive(ClapArgs)] @@ -74,6 +78,13 @@ struct RunArgs { dry_run: bool, } +#[derive(ClapArgs)] +struct VmLauncherArgs { + /// Path to the generated VM launch specification. + #[arg(long)] + spec: String, +} + async fn run_external_api(app: App, figment: Figment, api_auth: ApiToken) -> Result<()> { let version = app_version(); let openapi_doc = openapi::build_openapi_doc(&version)?; @@ -155,6 +166,12 @@ async fn main() -> Result<()> { let args = Args::parse(); + // The per-VM launcher must stay minimal: do not load or validate the VMM + // server configuration in this mode. + if let Some(Command::VmLauncher(launcher_args)) = &args.command { + return vm_launcher::run(Path::new(&launcher_args.spec)).await; + } + let figment = config::load_config_figment(args.config.as_deref()); let config = Config::extract_or_default(&figment)?.abs_path()?; @@ -166,6 +183,7 @@ async fn main() -> Result<()> { // Handle commands match args.command.unwrap_or_default() { + Command::VmLauncher(_) => unreachable!("launcher mode handled before config loading"), Command::Run(run_args) => { // One-shot VM execution mode return one_shot::run_one_shot( diff --git a/dstack/vmm/src/main_routes.rs b/dstack/vmm/src/main_routes.rs index c74e7e63b..13fcffa57 100644 --- a/dstack/vmm/src/main_routes.rs +++ b/dstack/vmm/src/main_routes.rs @@ -57,11 +57,6 @@ async fn beta(app: &State) -> (ContentType, String) { index(app).await } -#[get("/v0")] -async fn v0(app: &State) -> (ContentType, String) { - render_console(file_or_include_str!("console_v0.html"), app) -} - #[get("/res/")] async fn res(path: &str) -> Result<(ContentType, String), Custom> { match path { @@ -183,5 +178,5 @@ fn vm_logs( } pub fn routes() -> Vec { - routes![index, v1, beta, v0, res, vm_logs] + routes![index, v1, beta, res, vm_logs] } diff --git a/dstack/vmm/src/vm_launcher.rs b/dstack/vmm/src/vm_launcher.rs new file mode 100644 index 000000000..8499b775f --- /dev/null +++ b/dstack/vmm/src/vm_launcher.rs @@ -0,0 +1,293 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +use std::os::unix::process::CommandExt; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use anyhow::{bail, Context, Result}; +use serde::{Deserialize, Serialize}; +use tokio::process::{Child, Command}; +use tokio::signal::unix::{signal, SignalKind}; +use tokio::time::{sleep, timeout, Instant}; +use tracing::{error, info, warn}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ChildCommand { + pub command: String, + #[serde(default)] + pub args: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct LaunchSpec { + pub qemu: ChildCommand, + pub swtpm: ChildCommand, + pub swtpm_socket: PathBuf, + #[serde(default = "default_startup_timeout_ms")] + pub startup_timeout_ms: u64, + #[serde(default = "default_shutdown_timeout_ms")] + pub shutdown_timeout_ms: u64, +} + +fn default_startup_timeout_ms() -> u64 { + 5_000 +} + +fn default_shutdown_timeout_ms() -> u64 { + 10_000 +} + +struct SocketCleanup(PathBuf); + +impl Drop for SocketCleanup { + fn drop(&mut self) { + if self.0.exists() { + if let Err(error) = fs_err::remove_file(&self.0) { + warn!(path = %self.0.display(), %error, "failed to clean up swtpm socket"); + } + } + } +} + +fn spawn_child(spec: &ChildCommand) -> Result { + let parent = unsafe { libc::getpid() }; + let mut command = Command::new(&spec.command); + command.args(&spec.args); + // SAFETY: pre_exec only invokes async-signal-safe libc operations. Checking + // the parent after PR_SET_PDEATHSIG closes the fork/parent-exit race. + unsafe { + command.as_std_mut().pre_exec(move || { + if libc::setpgid(0, 0) != 0 { + return Err(std::io::Error::last_os_error()); + } + if libc::prctl(libc::PR_SET_PDEATHSIG, libc::SIGKILL) != 0 { + return Err(std::io::Error::last_os_error()); + } + if libc::getppid() != parent { + libc::raise(libc::SIGKILL); + } + Ok(()) + }); + } + command + .spawn() + .with_context(|| format!("failed to start {}", spec.command)) +} + +async fn stop_child(child: &mut Child, name: &str, grace: Duration) { + let Some(pid) = child.id() else { + return; + }; + // SAFETY: pid comes from the live Child handle and SIGTERM has no pointer arguments. + if unsafe { libc::kill(-(pid as libc::pid_t), libc::SIGTERM) } != 0 { + let error = std::io::Error::last_os_error(); + if error.raw_os_error() != Some(libc::ESRCH) { + warn!(%pid, %name, %error, "failed to terminate child"); + } + } + match timeout(grace, child.wait()).await { + Ok(Ok(status)) => info!(%pid, %name, %status, "child stopped"), + Ok(Err(error)) => warn!(%pid, %name, %error, "failed to wait for child"), + Err(_) => { + warn!(%pid, %name, "child did not stop gracefully; killing"); + if unsafe { libc::kill(-(pid as libc::pid_t), libc::SIGKILL) } != 0 { + let error = std::io::Error::last_os_error(); + if error.raw_os_error() != Some(libc::ESRCH) { + warn!(%pid, %name, %error, "failed to kill child process group"); + } + } + if let Err(error) = child.wait().await { + warn!(%pid, %name, %error, "failed to reap killed child"); + } + } + } +} + +async fn wait_for_swtpm(swtpm: &mut Child, socket: &Path, deadline: Instant) -> Result<()> { + loop { + if socket.exists() { + return Ok(()); + } + if Instant::now() >= deadline { + bail!("timed out waiting for swtpm socket {}", socket.display()); + } + tokio::select! { + status = swtpm.wait() => { + bail!("swtpm exited before socket was ready: {}", status?); + } + _ = sleep(Duration::from_millis(50)) => {} + } + } +} + +pub async fn run(spec_path: &Path) -> Result<()> { + let raw = fs_err::read(spec_path) + .with_context(|| format!("failed to read launch spec {}", spec_path.display()))?; + let spec: LaunchSpec = serde_json::from_slice(&raw).context("failed to parse launch spec")?; + let _socket_cleanup = SocketCleanup(spec.swtpm_socket.clone()); + if spec.swtpm_socket.exists() { + fs_err::remove_file(&spec.swtpm_socket).context("failed to remove stale swtpm socket")?; + } + + let grace = Duration::from_millis(spec.shutdown_timeout_ms); + let mut terminate = signal(SignalKind::terminate()).context("failed to watch SIGTERM")?; + let mut interrupt = signal(SignalKind::interrupt()).context("failed to watch SIGINT")?; + let mut swtpm = spawn_child(&spec.swtpm)?; + enum StartupExit { + Ready, + Error(anyhow::Error), + Signal, + } + let startup_exit = { + let startup = wait_for_swtpm( + &mut swtpm, + &spec.swtpm_socket, + Instant::now() + Duration::from_millis(spec.startup_timeout_ms), + ); + tokio::pin!(startup); + tokio::select! { + result = &mut startup => match result { + Ok(()) => StartupExit::Ready, + Err(error) => StartupExit::Error(error), + }, + _ = terminate.recv() => StartupExit::Signal, + _ = interrupt.recv() => StartupExit::Signal, + } + }; + match startup_exit { + StartupExit::Ready => {} + StartupExit::Error(error) => { + stop_child(&mut swtpm, "swtpm", grace).await; + return Err(error); + } + StartupExit::Signal => { + stop_child(&mut swtpm, "swtpm", grace).await; + return Ok(()); + } + } + + let mut qemu = match spawn_child(&spec.qemu) { + Ok(child) => child, + Err(error) => { + stop_child(&mut swtpm, "swtpm", grace).await; + return Err(error); + } + }; + info!( + qemu_pid = qemu.id(), + swtpm_pid = swtpm.id(), + "VM processes started" + ); + + enum Exit { + Signal, + Qemu(std::process::ExitStatus), + Swtpm(std::process::ExitStatus), + } + let exit = tokio::select! { + _ = terminate.recv() => Exit::Signal, + _ = interrupt.recv() => Exit::Signal, + status = qemu.wait() => Exit::Qemu(status.context("failed to wait for QEMU")?), + status = swtpm.wait() => Exit::Swtpm(status.context("failed to wait for swtpm")?), + }; + + match exit { + Exit::Signal => { + tokio::join!( + stop_child(&mut qemu, "qemu", grace), + stop_child(&mut swtpm, "swtpm", grace) + ); + Ok(()) + } + Exit::Qemu(status) => { + stop_child(&mut swtpm, "swtpm", grace).await; + if status.success() { + Ok(()) + } else { + bail!("QEMU exited with {status}") + } + } + Exit::Swtpm(status) => { + error!(%status, "swtpm exited while QEMU was running"); + stop_child(&mut qemu, "qemu", grace).await; + bail!("swtpm exited with {status}") + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::os::unix::net::UnixListener; + + fn shell(script: String) -> ChildCommand { + ChildCommand { + command: "/bin/sh".into(), + args: vec!["-c".into(), script], + } + } + + fn process_is_gone(pid: libc::pid_t) -> bool { + // SAFETY: signal 0 only probes whether the PID still exists. + let missing = unsafe { libc::kill(pid, 0) != 0 }; + missing && std::io::Error::last_os_error().raw_os_error() == Some(libc::ESRCH) + } + + async fn create_fake_socket(path: PathBuf) { + sleep(Duration::from_millis(100)).await; + let _listener = UnixListener::bind(path).unwrap(); + sleep(Duration::from_secs(5)).await; + } + + #[tokio::test] + async fn qemu_failure_stops_and_reaps_swtpm() { + let dir = tempfile::tempdir().unwrap(); + let socket = dir.path().join("swtpm.sock"); + let swtpm_pid = dir.path().join("swtpm.pid"); + let spec = LaunchSpec { + qemu: shell("exit 7".into()), + swtpm: shell(format!("echo $$ > {}; sleep 30", swtpm_pid.display())), + swtpm_socket: socket.clone(), + startup_timeout_ms: 2_000, + shutdown_timeout_ms: 500, + }; + let spec_path = dir.path().join("spec.json"); + fs_err::write(&spec_path, serde_json::to_vec(&spec).unwrap()).unwrap(); + tokio::spawn(create_fake_socket(socket)); + + assert!(run(&spec_path).await.is_err()); + let pid: libc::pid_t = fs_err::read_to_string(swtpm_pid) + .unwrap() + .trim() + .parse() + .unwrap(); + assert!(process_is_gone(pid)); + } + + #[tokio::test] + async fn swtpm_failure_stops_and_reaps_qemu() { + let dir = tempfile::tempdir().unwrap(); + let socket = dir.path().join("swtpm.sock"); + let qemu_pid = dir.path().join("qemu.pid"); + let spec = LaunchSpec { + qemu: shell(format!("echo $$ > {}; sleep 30", qemu_pid.display())), + swtpm: shell("sleep 0.2; exit 9".into()), + swtpm_socket: socket.clone(), + startup_timeout_ms: 2_000, + shutdown_timeout_ms: 500, + }; + let spec_path = dir.path().join("spec.json"); + fs_err::write(&spec_path, serde_json::to_vec(&spec).unwrap()).unwrap(); + tokio::spawn(create_fake_socket(socket)); + + assert!(run(&spec_path).await.is_err()); + let pid: libc::pid_t = fs_err::read_to_string(qemu_pid) + .unwrap() + .trim() + .parse() + .unwrap(); + assert!(process_is_gone(pid)); + } +} diff --git a/dstack/vmm/ui/README.md b/dstack/vmm/ui/README.md index 8ba28fb0b..0db86913a 100644 --- a/dstack/vmm/ui/README.md +++ b/dstack/vmm/ui/README.md @@ -15,8 +15,8 @@ npm run watch `dstack-vmm` now builds the single-file HTML artifact from `build.rs` and writes it to Cargo's `OUT_DIR`. This requires Node.js and npm to be installed; if they are -missing, the Rust build will fail with an installation hint. The previous -`console_v0.html` remains untouched so the legacy UI stays available under `/v0`. +missing, the Rust build will fail with an installation hint. The legacy +`console_v0.html` route has been removed. The UI codebase is written in TypeScript. The build pipeline performs three steps: diff --git a/dstack/vmm/ui/src/components/CreateVmDialog.ts b/dstack/vmm/ui/src/components/CreateVmDialog.ts index c0e59eb20..0eabb9cd3 100644 --- a/dstack/vmm/ui/src/components/CreateVmDialog.ts +++ b/dstack/vmm/ui/src/components/CreateVmDialog.ts @@ -141,11 +141,11 @@ const CreateVmDialogComponent = { - + -
+
@@ -184,7 +184,7 @@ const CreateVmDialogComponent = { - +
diff --git a/dstack/vmm/ui/src/composables/useVmManager.ts b/dstack/vmm/ui/src/composables/useVmManager.ts index c6c60c741..609de8aa5 100644 --- a/dstack/vmm/ui/src/composables/useVmManager.ts +++ b/dstack/vmm/ui/src/composables/useVmManager.ts @@ -813,7 +813,9 @@ type CreateVmPayloadSource = { public_logs: vmForm.value.public_logs, public_sysinfo: vmForm.value.public_sysinfo, public_tcbinfo: vmForm.value.public_tcbinfo, - key_provider_id: vmForm.value.key_provider_id, + key_provider_id: vmForm.value.key_provider === 'kms' || vmForm.value.key_provider === 'local' + ? vmForm.value.key_provider_id + : '', allowed_envs: vmForm.value.encryptedEnvs.map((env) => env.key), no_instance_id: !vmForm.value.gateway_enabled, secure_time: false, @@ -1360,11 +1362,6 @@ type CreateVmPayloadSource = { window.open('/api-docs/docs', '_blank', 'noopener'); } - function openLegacyUi() { - closeSystemMenu(); - window.open('/v0', '_blank', 'noopener'); - } - function shortUptime(uptime?: string | null) { if (!uptime) { return '-'; @@ -1807,7 +1804,6 @@ type CreateVmPayloadSource = { toggleSystemMenu, closeSystemMenu, openApiDocs, - openLegacyUi, reloadVMs, devMode, toggleDevMode, diff --git a/dstack/vmm/ui/src/templates/app.html b/dstack/vmm/ui/src/templates/app.html index 7e38806c7..139155032 100644 --- a/dstack/vmm/ui/src/templates/app.html +++ b/dstack/vmm/ui/src/templates/app.html @@ -57,13 +57,6 @@

dstack-vmm

API Docs -