From 5c1daaa46a371640e2fa56a72ad81e6dd41d4422 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 19 Jul 2026 20:49:31 -0700 Subject: [PATCH 01/12] feat(vmm): add swtpm key provider for no-TEE dev VMs --- CONTRIBUTING.md | 6 ++ docs/security/cvm-boundaries.md | 2 +- dstack/tpm-attest/src/lib.rs | 12 ++- dstack/vmm/src/app.rs | 54 +++++++++++-- dstack/vmm/src/app/qemu.rs | 135 ++++++++++++++++++++++++++------ dstack/vmm/src/app/workdir.rs | 8 ++ dstack/vmm/src/console_v0.html | 41 +++++----- 7 files changed, 208 insertions(+), 50 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5b69bce63..02edd7d2d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -62,6 +62,12 @@ dstack deploy ./docker-compose.yml \ --no-tee ``` +To exercise persistent TPM-backed app keys as well, install `swtpm` on the VMM +host and select `key_provider=tpm` in the VMM console (or pass `--key-provider +tpm` in tooling that exposes the compose option). The VMM keeps the software +TPM state in the VM work directory so that seal/unseal survives guest restarts. +This development provider is available only to no-TEE development images. + The simulator package is installed only in development images. This mode provides no hardware isolation and must never be used with production workloads or secrets. Real quote generation, hardware isolation, and KMS diff --git a/docs/security/cvm-boundaries.md b/docs/security/cvm-boundaries.md index e35496727..aea673f5f 100644 --- a/docs/security/cvm-boundaries.md +++ b/docs/security/cvm-boundaries.md @@ -44,7 +44,7 @@ This is the main configuration file for the application in JSON format: | init_script | 0.5.5 | string | Bash script that executed prior to dockerd startup | | storage_fs | 0.5.5 | string | Filesystem type for the data disk of the CVM. Supported values: "zfs", "ext4". default to "zfs". **ZFS:** Ensures filesystem integrity with built-in data protection features. **ext4:** Provides better performance for database applications with lower overhead and faster I/O operations, but no strong integrity protection. | | swap_size | 0.5.5 | string/integer | The linux swap size. default to 0. Can be in byte or human-readable format (e.g., "1G", "256M"). | -| key_provider | 0.5.6 | string | Key provider type. Supported values: "none", "kms", "local", "tpm". `"tpm"` is only supported on platforms whose TPM is part of the platform trust model (GCP vTPM, AWS EC2 NitroTPM); on other platforms guest setup fails closed, since sealing to a host-provided software TPM (e.g. swtpm under bare QEMU) offers no protection against the host. | +| key_provider | 0.5.6 | string | Key provider type. Supported values: "none", "kms", "local", "tpm". GCP vTPM and AWS EC2 NitroTPM are part of their platform trust models. The Dstack development platform can use VMM-managed swtpm to exercise seal/unseal and restart persistence, but it offers no protection against the host and is intentionally not accepted by remote verifiers. | The hash of this file content is extended as the dstack `compose-hash` launch event. On TDX-family platforms the launch event is measured into RTMR3. On AWS NitroTPM it is measured into non-resettable SHA384 PCR14 before the `system-ready` launch boundary. Remote verifiers extract and replay this event during attestation. diff --git a/dstack/tpm-attest/src/lib.rs b/dstack/tpm-attest/src/lib.rs index 7105af79d..a0f3046a2 100644 --- a/dstack/tpm-attest/src/lib.rs +++ b/dstack/tpm-attest/src/lib.rs @@ -48,7 +48,7 @@ const AWS_NITRO_PCRS: [u32; 5] = [4, 7, 8, 12, 14]; pub fn dstack_pcr_policy_for_platform(platform: Platform) -> Result { match platform { - Platform::Gcp => Ok(dstack_pcr_policy()), + Platform::Dstack | Platform::Gcp => Ok(dstack_pcr_policy()), Platform::AwsEc2 => Ok(PcrSelection::sha384(&AWS_NITRO_PCRS)), _ => bail!("TPM local key provider is not supported on {platform:?}"), } @@ -367,6 +367,16 @@ mod tests { let policy = dstack_pcr_policy_for_platform(Platform::AwsEc2).unwrap(); assert_eq!(policy.to_arg(), "sha384:4,7,8,12,14"); } + + #[test] + fn dstack_platform_uses_development_tpm_policy() { + assert_eq!( + dstack_pcr_policy_for_platform(Platform::Dstack) + .unwrap() + .to_arg(), + "sha256:0,2,14" + ); + } } mod gcp_ak; diff --git a/dstack/vmm/src/app.rs b/dstack/vmm/src/app.rs index f4dcd0d12..a05605d31 100644 --- a/dstack/vmm/src/app.rs +++ b/dstack/vmm/src/app.rs @@ -399,6 +399,7 @@ impl App { } for process in processes { if let Err(err) = self.supervisor.deploy(&process).await { + self.stop_dependent_processes(id).await; if let Err(clear_err) = work_dir.clear_runtime_networks() { warn!( id, @@ -430,8 +431,24 @@ impl App { pub async fn stop_vm(&self, id: &str) -> Result<()> { self.set_started(id, false)?; self.cleanup_port_forward(id).await; - self.supervisor.stop(id).await?; - Ok(()) + let result = self.supervisor.stop(id).await; + self.stop_dependent_processes(id).await; + result + } + + async fn stop_dependent_processes(&self, id: &str) { + for process in self.supervisor.list().await.unwrap_or_default() { + let note = + serde_json::from_str::(&process.config.note).unwrap_or_default(); + if note.live_for.as_deref() == Some(id) { + if let Err(err) = self.supervisor.stop(&process.config.id).await { + debug!( + process_id = process.config.id, + "failed to stop dependent process: {err:?}" + ); + } + } + } } pub async fn remove_vm(&self, id: &str) -> Result<()> { @@ -475,6 +492,7 @@ impl App { if let Err(err) = self.supervisor.stop(id).await { debug!("supervisor.stop({id}) during removal: {err:?}"); } + self.stop_dependent_processes(id).await; // Poll until the process is no longer running, then remove it. // Some VMs take a long time to stop (e.g. 2+ hours), so we wait indefinitely. @@ -509,6 +527,31 @@ impl App { } } + // Auxiliary processes (currently swtpm) are owned by the VM through + // ProcessAnnotation::live_for. Remove their stopped supervisor entries + // before deleting the VM workdir. + for process in self.supervisor.list().await.unwrap_or_default() { + let note = + serde_json::from_str::(&process.config.note).unwrap_or_default(); + if note.live_for.as_deref() != Some(id) { + continue; + } + for _ in 0..50 { + match self.supervisor.info(&process.config.id).await { + Ok(Some(info)) if info.state.status.is_running() => { + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + } + _ => break, + } + } + if let Err(err) = self.supervisor.remove(&process.config.id).await { + warn!( + process_id = process.config.id, + "failed to remove dependent process: {err:?}" + ); + } + } + // Only delete the workdir for user-initiated removal or if .removing marker exists. // Orphaned supervisor processes without the marker keep their data intact. let vm_path = self.work_dir(id); @@ -748,13 +791,14 @@ impl App { // Clean up orphaned supervisor processes (in supervisor but not loaded as VMs) let loaded_vm_ids: HashSet = self.lock().vms.keys().cloned().collect(); - for (_, process) in &running_vms { - if !loaded_vm_ids.contains(&process.config.id) { + for (note, process) in &running_vms { + let owner = note.live_for.as_deref().unwrap_or(&process.config.id); + if !loaded_vm_ids.contains(owner) { info!( "Cleaning up orphaned supervisor process: {}", process.config.id ); - self.spawn_finish_remove(&process.config.id); + self.spawn_finish_remove(owner); } } diff --git a/dstack/vmm/src/app/qemu.rs b/dstack/vmm/src/app/qemu.rs index 6850bb653..db9432645 100644 --- a/dstack/vmm/src/app/qemu.rs +++ b/dstack/vmm/src/app/qemu.rs @@ -179,7 +179,8 @@ struct PreparedQemuLaunch { hugepage_numa_nodes: Option>, gpu_numa_nodes: HashMap, numa_cpus: Option, - tpm_path: Option<&'static str>, + swtpm_socket: Option, + swtpm_path: Option, tdx_mr_config_id: Option, snp_host_data: Option, snp_launch_params: Option, @@ -219,11 +220,23 @@ impl PreparedQemuLaunch { } else { None }; - let tpm_path = if matches!(app_compose.key_provider(), KeyProviderKind::Tpm) { - Some(detect_tpm_device()?) - } else { - None - }; + let (swtpm_socket, swtpm_path) = + if matches!(app_compose.key_provider(), KeyProviderKind::Tpm) { + if !vm.manifest.no_tee || !vm.image.info.is_dev { + bail!("swtpm key provider is only available to no-TEE development VMs"); + } + let swtpm_path = which::which("swtpm") + .context("tpm key provider requested but swtpm is not installed")?; + let state_dir = workdir.swtpm_state_dir(); + fs::create_dir_all(&state_dir).context("failed to create swtpm state directory")?; + let socket = workdir.swtpm_socket(); + if socket.exists() { + fs::remove_file(&socket).context("failed to remove stale swtpm socket")?; + } + (Some(socket), Some(swtpm_path)) + } else { + (None, None) + }; prepare_shared_disk(&workdir, cfg)?; let tee_enabled = !vm.manifest.no_tee; @@ -257,7 +270,8 @@ impl PreparedQemuLaunch { hugepage_numa_nodes, gpu_numa_nodes, numa_cpus, - tpm_path, + swtpm_socket, + swtpm_path, tdx_mr_config_id, snp_host_data, snp_launch_params, @@ -301,16 +315,6 @@ fn prepare_shared_disk(workdir: &VmWorkDir, cfg: &CvmConfig) -> Result<()> { create_shared_disk(&shared_disk_path, shared_dir).context("failed to create shared disk") } -fn detect_tpm_device() -> Result<&'static str> { - if Path::new("/dev/tpmrm0").exists() { - Ok("/dev/tpmrm0") - } else if Path::new("/dev/tpm0").exists() { - Ok("/dev/tpm0") - } else { - bail!("tpm key provider requested but no TPM device found on host") - } -} - struct QemuCommandBuilder<'a> { vm: &'a VmConfig, cfg: &'a CvmConfig, @@ -326,14 +330,76 @@ impl VmConfig { gpus: &GpuConfig, ) -> Result> { let prepared = PreparedQemuLaunch::prepare(self, workdir, cfg, gpus)?; - let process = QemuCommandBuilder { + let mut process = QemuCommandBuilder { vm: self, cfg, gpus, prepared: &prepared, } .build()?; - Ok(vec![process]) + let Some(socket) = prepared.swtpm_socket.as_deref() else { + return Ok(vec![process]); + }; + let swtpm_path = prepared + .swtpm_path + .as_ref() + .context("missing swtpm executable for configured socket")?; + + // The supervisor starts process configs in order, but swtpm may need a + // moment to create its socket. Make QEMU wait without shell-quoting any + // of its arguments: they are passed as positional parameters. + let original_command = std::mem::replace(&mut process.command, "/bin/sh".into()); + let original_args = std::mem::take(&mut process.args); + process.args = vec![ + "-c".into(), + "for i in $(seq 1 100); do [ -S \"$1\" ] && shift && exec \"$@\"; sleep 0.05; done; echo 'timed out waiting for swtpm socket' >&2; exit 1".into(), + "dstack-qemu-wait-swtpm".into(), + socket.to_string_lossy().into_owned(), + original_command, + ]; + process.args.extend(original_args); + + let note = serde_json::to_string(&ProcessAnnotation { + kind: "swtpm".into(), + live_for: Some(self.manifest.id.clone()), + })?; + let swtpm_args = vec![ + "socket".into(), + "--tpm2".into(), + "--tpmstate".into(), + format!("dir={}", prepared.workdir.swtpm_state_dir().display()), + "--ctrl".into(), + format!("type=unixio,path={},mode=0666", socket.display()), + "--flags".into(), + "not-need-init,startup-clear".into(), + ]; + let swtpm = ProcessConfig { + id: format!("{}-swtpm", self.manifest.id), + name: format!("{} swtpm", self.manifest.name), + command: swtpm_path.to_string_lossy().into_owned(), + args: swtpm_args, + env: Default::default(), + cwd: prepared.workdir.path().to_string_lossy().into_owned(), + stdout: prepared + .workdir + .stdout_file() + .to_string_lossy() + .into_owned(), + stderr: prepared + .workdir + .stderr_file() + .to_string_lossy() + .into_owned(), + pidfile: prepared + .workdir + .swtpm_state_dir() + .join("supervisor.pid") + .to_string_lossy() + .into_owned(), + cid: None, + note, + }; + Ok(vec![swtpm, process]) } } @@ -518,10 +584,12 @@ impl QemuCommandBuilder<'_> { } fn configure_tpm_and_vsock(&self, command: &mut Command) { - if let Some(tpm_path) = self.prepared.tpm_path { + if let Some(socket) = &self.prepared.swtpm_socket { command + .arg("-chardev") + .arg(format!("socket,id=chrtpm,path={}", socket.display())) .arg("-tpmdev") - .arg(format!("passthrough,id=tpm0,path={tpm_path}")) + .arg("emulator,id=tpm0,chardev=chrtpm") .arg("-device") .arg("tpm-tis,tpmdev=tpm0"); } @@ -970,14 +1038,15 @@ mod tests { workdir: PathBuf::from("/does-not-exist/vm-1"), gateway_enabled: false, }; - let prepared = PreparedQemuLaunch { + let mut prepared = PreparedQemuLaunch { workdir: VmWorkDir::new("/does-not-exist/vm-1"), platform: TeePlatform::Tdx, networks: vec![config.cvm.networking.clone(), config.cvm.networking.clone()], hugepage_numa_nodes: None, gpu_numa_nodes: HashMap::new(), numa_cpus: None, - tpm_path: None, + swtpm_socket: None, + swtpm_path: None, tdx_mr_config_id: None, snp_host_data: None, snp_launch_params: None, @@ -1024,5 +1093,25 @@ mod tests { .args .iter() .any(|arg| arg.contains("virtio-net-pci,netdev=net1"))); + + prepared.swtpm_socket = Some(PathBuf::from("/does-not-exist/vm-1/swtpm/swtpm.sock")); + let process = QemuCommandBuilder { + vm: &vm, + cfg: &config.cvm, + gpus: &GpuConfig::default(), + prepared: &prepared, + } + .build() + .unwrap(); + assert!(process.args.windows(2).any(|args| { + args == [ + "-chardev", + "socket,id=chrtpm,path=/does-not-exist/vm-1/swtpm/swtpm.sock", + ] + })); + assert!(process + .args + .windows(2) + .any(|args| args == ["-tpmdev", "emulator,id=tpm0,chardev=chrtpm"])); } } diff --git a/dstack/vmm/src/app/workdir.rs b/dstack/vmm/src/app/workdir.rs index 6984cd4b5..b3ce4f613 100644 --- a/dstack/vmm/src/app/workdir.rs +++ b/dstack/vmm/src/app/workdir.rs @@ -104,6 +104,14 @@ impl VmWorkDir { self.workdir.join("shared") } + pub fn swtpm_state_dir(&self) -> PathBuf { + self.workdir.join("swtpm") + } + + pub fn swtpm_socket(&self) -> PathBuf { + self.swtpm_state_dir().join("swtpm.sock") + } + pub fn app_compose_path(&self) -> PathBuf { self.shared_dir().join(APP_COMPOSE) } diff --git a/dstack/vmm/src/console_v0.html b/dstack/vmm/src/console_v0.html index d68227724..c814a9432 100644 --- a/dstack/vmm/src/console_v0.html +++ b/dstack/vmm/src/console_v0.html @@ -798,13 +798,14 @@

Deploy a new instance

- -
-
+
-
+
@@ -849,12 +850,12 @@

Deploy a new instance

style="width: 16px; height: 16px;">
-
+
-
+
Derive VM }, storage_fs: '', app_id: '', - kms_enabled: true, - local_key_provider_enabled: false, + key_provider: 'kms', key_provider_id: '', gateway_enabled: true, public_logs: true, @@ -1743,13 +1743,15 @@

Derive VM

"username": vmForm.value.docker_config.username, "token_key": vmForm.value.docker_config.token_key, } : {}, - "kms_enabled": vmForm.value.kms_enabled, + "key_provider": vmForm.value.key_provider, + "kms_enabled": vmForm.value.key_provider === 'kms', "gateway_enabled": vmForm.value.gateway_enabled, "public_logs": vmForm.value.public_logs, "public_sysinfo": vmForm.value.public_sysinfo, "public_tcbinfo": vmForm.value.public_tcbinfo, - "local_key_provider_enabled": vmForm.value.local_key_provider_enabled, - "key_provider_id": vmForm.value.key_provider_id, + "local_key_provider_enabled": vmForm.value.key_provider === 'local', + "key_provider_id": (vmForm.value.key_provider === 'kms' || vmForm.value.key_provider === 'local') + ? vmForm.value.key_provider_id : '', "allowed_envs": vmForm.value.encryptedEnvs.map(env => env.key), "no_instance_id": !vmForm.value.gateway_enabled, "secure_time": false, @@ -1778,7 +1780,7 @@

Derive VM

if (imgFeatures.compose_version < 2) { // Compatibility with old images const features = []; - if (vmForm.value.kms_enabled) features.push("kms"); + if (vmForm.value.key_provider === 'kms') features.push("kms"); if (vmForm.value.gateway_enabled) features.push("tproxy-net"); app_compose.features = features; app_compose.manifest_version = 1; @@ -1949,7 +1951,7 @@

Derive VM

allowed_envs: vmForm.value.encryptedEnvs.map(env => env.key), encrypted_env: await makeEncryptedEnv( vmForm.value.encryptedEnvs, - vmForm.value.kms_enabled, + vmForm.value.key_provider === 'kms', vmForm.value.app_id ), user_config: vmForm.value.user_config, @@ -2517,12 +2519,11 @@

Derive VM

() => vmForm.value.name, () => vmForm.value.dockerComposeFile, () => vmForm.value.preLaunchScript, - () => vmForm.value.kms_enabled, + () => vmForm.value.key_provider, () => vmForm.value.gateway_enabled, () => vmForm.value.public_logs, () => vmForm.value.public_sysinfo, () => vmForm.value.public_tcbinfo, - () => vmForm.value.local_key_provider_enabled, () => vmForm.value.key_provider_id, () => vmForm.value.docker_config.enabled, () => vmForm.value.docker_config.username, From 6fbd1ce5e640f34a281134c99228690a0f67d885 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 19 Jul 2026 20:58:01 -0700 Subject: [PATCH 02/12] fix(vmm): bind swtpm lifecycle to its VM --- dstack/vmm/src/app.rs | 132 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 132 insertions(+) diff --git a/dstack/vmm/src/app.rs b/dstack/vmm/src/app.rs index a05605d31..a3bd5f504 100644 --- a/dstack/vmm/src/app.rs +++ b/dstack/vmm/src/app.rs @@ -255,6 +255,26 @@ pub struct App { pub(crate) pull_status: Arc>>, } +#[derive(Debug, PartialEq, Eq)] +enum DependencyAction { + KeepRunning, + StopDependents, + StopVm, + Finished, +} + +fn dependency_action(main_running: bool, dependents_running: &[bool]) -> DependencyAction { + if dependents_running.is_empty() { + DependencyAction::Finished + } else if !main_running { + DependencyAction::StopDependents + } else if dependents_running.iter().any(|running| !running) { + DependencyAction::StopVm + } else { + DependencyAction::KeepRunning + } +} + impl App { pub(crate) fn lock(&self) -> MutexGuard<'_, AppState> { self.state.lock().or_panic("mutex poisoned") @@ -278,6 +298,7 @@ impl App { cid_pool, vms: HashMap::new(), active_forwards: HashMap::new(), + dependency_monitors: HashSet::new(), })), config: Arc::new(config), forward_service: Arc::new(tokio::sync::Mutex::new(ForwardService::new())), @@ -376,6 +397,7 @@ impl App { vm_state.config.clone() }; if !is_running { + self.wait_for_dependent_processes_stopped(id).await?; let work_dir = self.work_dir(id); for path in [work_dir.serial_pty(), work_dir.qmp_socket()] { if path.symlink_metadata().is_ok() { @@ -397,6 +419,7 @@ impl App { let vm_state = state.get_mut(id).context("VM not found")?; vm_state.state.runtime_networks = runtime_networks; } + let has_dependent_processes = processes.len() > 1; for process in processes { if let Err(err) = self.supervisor.deploy(&process).await { self.stop_dependent_processes(id).await; @@ -413,6 +436,9 @@ impl App { .with_context(|| format!("failed to start process {}", process.id)); } } + if has_dependent_processes { + self.spawn_dependency_monitor(id); + } let mut state = self.lock(); let vm_state = state.get_mut(id).context("VM not found")?; @@ -451,6 +477,85 @@ impl App { } } + async fn wait_for_dependent_processes_stopped(&self, id: &str) -> Result<()> { + for _ in 0..50 { + let running = self.supervisor.list().await?.into_iter().any(|process| { + let note = serde_json::from_str::(&process.config.note) + .unwrap_or_default(); + note.live_for.as_deref() == Some(id) && process.state.status.is_running() + }); + if !running { + return Ok(()); + } + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + } + bail!("timed out waiting for dependent processes of VM {id} to stop") + } + + /// Keep auxiliary processes fail-closed with their owning VM. If QEMU + /// exits, all dependents are stopped. If a required dependent exits first, + /// QEMU is stopped rather than continuing with a broken emulated device. + fn spawn_dependency_monitor(&self, id: &str) { + { + let mut state = self.lock(); + if !state.dependency_monitors.insert(id.to_string()) { + return; + } + } + let app = self.clone(); + let id = id.to_string(); + tokio::spawn(async move { + loop { + tokio::time::sleep(std::time::Duration::from_millis(250)).await; + let processes = match app.supervisor.list().await { + Ok(processes) => processes, + Err(err) => { + warn!(vm_id = id, "failed to inspect dependent processes: {err:?}"); + continue; + } + }; + let main_running = processes + .iter() + .find(|process| process.config.id == id) + .is_some_and(|process| process.state.status.is_running()); + let dependents = processes + .iter() + .filter(|process| { + serde_json::from_str::(&process.config.note) + .unwrap_or_default() + .live_for + .as_deref() + == Some(id.as_str()) + }) + .collect::>(); + let dependent_states = dependents + .iter() + .map(|process| process.state.status.is_running()) + .collect::>(); + match dependency_action(main_running, &dependent_states) { + DependencyAction::KeepRunning => continue, + DependencyAction::Finished => break, + DependencyAction::StopDependents => { + app.stop_dependent_processes(&id).await; + break; + } + DependencyAction::StopVm => { + error!(vm_id = id, "required VM process exited; stopping VM"); + if let Err(err) = app.supervisor.stop(&id).await { + debug!( + vm_id = id, + "failed to stop VM after dependent exit: {err:?}" + ); + } + app.stop_dependent_processes(&id).await; + break; + } + } + } + app.lock().dependency_monitors.remove(&id); + }); + } + pub async fn remove_vm(&self, id: &str) -> Result<()> { { let mut state = self.lock(); @@ -802,6 +907,13 @@ impl App { } } + // Re-establish lifecycle monitoring after a VMM restart. The monitor + // also reconciles a QEMU or dependent process that exited while VMM was + // unavailable. + for id in self.lock().vms.keys().cloned().collect::>() { + self.spawn_dependency_monitor(&id); + } + // Restore port forwarding for running bridge-mode VMs with persisted guest IPs let vm_ids: Vec = self.lock().vms.keys().cloned().collect(); for id in vm_ids { @@ -1593,6 +1705,24 @@ mod tests { hex::encode(vec![byte; len]) } + #[test] + fn dependency_lifecycle_is_fail_closed() { + assert_eq!( + dependency_action(true, &[true]), + DependencyAction::KeepRunning + ); + assert_eq!( + dependency_action(false, &[true]), + DependencyAction::StopDependents + ); + assert_eq!(dependency_action(true, &[false]), DependencyAction::StopVm); + assert_eq!( + dependency_action(false, &[false]), + DependencyAction::StopDependents + ); + assert_eq!(dependency_action(true, &[]), DependencyAction::Finished); + } + #[test] fn gpu_config_has_gpus_only_when_resolved_gpu_list_is_non_empty() { assert!(!GpuConfig::default().has_gpus()); @@ -2194,6 +2324,8 @@ pub(crate) struct AppState { vms: HashMap, /// Tracks active port forwarding rules per VM ID (bridge mode only). active_forwards: HashMap>, + /// VM IDs with an active auxiliary-process lifecycle monitor. + dependency_monitors: HashSet, } impl AppState { From 013bd47bd211b7414f4f0cf89912dc33b08741bc Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 20 Jul 2026 11:59:41 +0800 Subject: [PATCH 03/12] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- dstack/vmm/src/app/qemu.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dstack/vmm/src/app/qemu.rs b/dstack/vmm/src/app/qemu.rs index db9432645..acbdb184c 100644 --- a/dstack/vmm/src/app/qemu.rs +++ b/dstack/vmm/src/app/qemu.rs @@ -369,7 +369,7 @@ impl VmConfig { "--tpmstate".into(), format!("dir={}", prepared.workdir.swtpm_state_dir().display()), "--ctrl".into(), - format!("type=unixio,path={},mode=0666", socket.display()), + format!("type=unixio,path={},mode=0600", socket.display()), "--flags".into(), "not-need-init,startup-clear".into(), ]; From 00c5103cdfc76e1899763586aea99fe6df805085 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 19 Jul 2026 21:35:30 -0700 Subject: [PATCH 04/12] refactor(vmm): supervise QEMU and swtpm with VM launcher --- dstack/Cargo.lock | 2 + dstack/vmm/Cargo.toml | 2 + dstack/vmm/src/app.rs | 232 ++++++------------------- dstack/vmm/src/app/qemu.rs | 82 ++++----- dstack/vmm/src/app/workdir.rs | 4 + dstack/vmm/src/main.rs | 18 ++ dstack/vmm/src/vm_launcher.rs | 310 ++++++++++++++++++++++++++++++++++ 7 files changed, 427 insertions(+), 223 deletions(-) create mode 100644 dstack/vmm/src/vm_launcher.rs diff --git a/dstack/Cargo.lock b/dstack/Cargo.lock index 87264d02a..c5d4c6580 100644 --- a/dstack/Cargo.lock +++ b/dstack/Cargo.lock @@ -2147,6 +2147,7 @@ dependencies = [ "humantime", "insta", "key-provider-client", + "libc", "load_config", "lspci", "nix 0.29.0", @@ -2169,6 +2170,7 @@ dependencies = [ "supervisor-client", "tailf", "tar", + "tempfile", "tokio", "tracing", "tracing-subscriber", diff --git a/dstack/vmm/Cargo.toml b/dstack/vmm/Cargo.toml index 9dbc6399c..4b1c1f144 100644 --- a/dstack/vmm/Cargo.toml +++ b/dstack/vmm/Cargo.toml @@ -15,6 +15,7 @@ rocket-vsock-listener = { workspace = true } tracing.workspace = true tracing-subscriber = { workspace = true, features = ["env-filter"] } anyhow.workspace = true +libc.workspace = true serde = { workspace = true, features = ["derive"] } serde_json.workspace = true shared_child.workspace = true @@ -64,6 +65,7 @@ tar.workspace = true [dev-dependencies] insta.workspace = true +tempfile.workspace = true [build-dependencies] or-panic.workspace = true diff --git a/dstack/vmm/src/app.rs b/dstack/vmm/src/app.rs index a3bd5f504..5c9ea15ae 100644 --- a/dstack/vmm/src/app.rs +++ b/dstack/vmm/src/app.rs @@ -48,6 +48,30 @@ pub(crate) mod registry; mod vm_info; mod workdir; +fn signal_pidfd(pid: u32, signal: libc::c_int) -> std::io::Result<()> { + // SAFETY: pidfd syscalls receive scalar arguments and a null siginfo pointer. + let fd = unsafe { libc::syscall(libc::SYS_pidfd_open, pid, 0) }; + if fd < 0 { + return Err(std::io::Error::last_os_error()); + } + let result = unsafe { + libc::syscall( + libc::SYS_pidfd_send_signal, + fd, + signal, + std::ptr::null::(), + 0, + ) + }; + // SAFETY: fd was returned by pidfd_open and is owned by this function. + unsafe { libc::close(fd as libc::c_int) }; + if result < 0 { + Err(std::io::Error::last_os_error()) + } else { + Ok(()) + } +} + #[derive(Deserialize, Serialize, Debug, Clone)] pub struct PortMapping { pub address: IpAddr, @@ -255,26 +279,6 @@ pub struct App { pub(crate) pull_status: Arc>>, } -#[derive(Debug, PartialEq, Eq)] -enum DependencyAction { - KeepRunning, - StopDependents, - StopVm, - Finished, -} - -fn dependency_action(main_running: bool, dependents_running: &[bool]) -> DependencyAction { - if dependents_running.is_empty() { - DependencyAction::Finished - } else if !main_running { - DependencyAction::StopDependents - } else if dependents_running.iter().any(|running| !running) { - DependencyAction::StopVm - } else { - DependencyAction::KeepRunning - } -} - impl App { pub(crate) fn lock(&self) -> MutexGuard<'_, AppState> { self.state.lock().or_panic("mutex poisoned") @@ -298,7 +302,6 @@ impl App { cid_pool, vms: HashMap::new(), active_forwards: HashMap::new(), - dependency_monitors: HashSet::new(), })), config: Arc::new(config), forward_service: Arc::new(tokio::sync::Mutex::new(ForwardService::new())), @@ -397,7 +400,6 @@ impl App { vm_state.config.clone() }; if !is_running { - self.wait_for_dependent_processes_stopped(id).await?; let work_dir = self.work_dir(id); for path in [work_dir.serial_pty(), work_dir.qmp_socket()] { if path.symlink_metadata().is_ok() { @@ -419,10 +421,8 @@ impl App { let vm_state = state.get_mut(id).context("VM not found")?; vm_state.state.runtime_networks = runtime_networks; } - let has_dependent_processes = processes.len() > 1; for process in processes { if let Err(err) = self.supervisor.deploy(&process).await { - self.stop_dependent_processes(id).await; if let Err(clear_err) = work_dir.clear_runtime_networks() { warn!( id, @@ -436,9 +436,6 @@ impl App { .with_context(|| format!("failed to start process {}", process.id)); } } - if has_dependent_processes { - self.spawn_dependency_monitor(id); - } let mut state = self.lock(); let vm_state = state.get_mut(id).context("VM not found")?; @@ -457,103 +454,36 @@ impl App { pub async fn stop_vm(&self, id: &str) -> Result<()> { self.set_started(id, false)?; self.cleanup_port_forward(id).await; - let result = self.supervisor.stop(id).await; - self.stop_dependent_processes(id).await; - result - } - - async fn stop_dependent_processes(&self, id: &str) { - for process in self.supervisor.list().await.unwrap_or_default() { - let note = - serde_json::from_str::(&process.config.note).unwrap_or_default(); - if note.live_for.as_deref() == Some(id) { - if let Err(err) = self.supervisor.stop(&process.config.id).await { - debug!( - process_id = process.config.id, - "failed to stop dependent process: {err:?}" - ); - } - } - } - } - - async fn wait_for_dependent_processes_stopped(&self, id: &str) -> Result<()> { - for _ in 0..50 { - let running = self.supervisor.list().await?.into_iter().any(|process| { - let note = serde_json::from_str::(&process.config.note) - .unwrap_or_default(); - note.live_for.as_deref() == Some(id) && process.state.status.is_running() - }); - if !running { - return Ok(()); - } - tokio::time::sleep(std::time::Duration::from_millis(100)).await; - } - bail!("timed out waiting for dependent processes of VM {id} to stop") + self.stop_vm_process(id).await?; + Ok(()) } - /// Keep auxiliary processes fail-closed with their owning VM. If QEMU - /// exits, all dependents are stopped. If a required dependent exits first, - /// QEMU is stopped rather than continuing with a broken emulated device. - fn spawn_dependency_monitor(&self, id: &str) { - { - let mut state = self.lock(); - if !state.dependency_monitors.insert(id.to_string()) { - return; + async fn stop_vm_process(&self, id: &str) -> Result<()> { + let Some(info) = self.supervisor.info(id).await? else { + return Ok(()); + }; + if info.state.status.is_running() { + let pid = info.state.pid.context("running VM launcher has no PID")?; + if let Err(error) = signal_pidfd(pid, libc::SIGTERM) { + warn!(id, %pid, %error, "failed to signal VM launcher gracefully; forcing shutdown"); + return self.supervisor.stop(id).await; } - } - let app = self.clone(); - let id = id.to_string(); - tokio::spawn(async move { - loop { - tokio::time::sleep(std::time::Duration::from_millis(250)).await; - let processes = match app.supervisor.list().await { - Ok(processes) => processes, - Err(err) => { - warn!(vm_id = id, "failed to inspect dependent processes: {err:?}"); - continue; - } - }; - let main_running = processes - .iter() - .find(|process| process.config.id == id) - .is_some_and(|process| process.state.status.is_running()); - let dependents = processes - .iter() - .filter(|process| { - serde_json::from_str::(&process.config.note) - .unwrap_or_default() - .live_for - .as_deref() - == Some(id.as_str()) - }) - .collect::>(); - let dependent_states = dependents - .iter() - .map(|process| process.state.status.is_running()) - .collect::>(); - match dependency_action(main_running, &dependent_states) { - DependencyAction::KeepRunning => continue, - DependencyAction::Finished => break, - DependencyAction::StopDependents => { - app.stop_dependent_processes(&id).await; - break; - } - DependencyAction::StopVm => { - error!(vm_id = id, "required VM process exited; stopping VM"); - if let Err(err) = app.supervisor.stop(&id).await { - debug!( - vm_id = id, - "failed to stop VM after dependent exit: {err:?}" - ); - } - app.stop_dependent_processes(&id).await; - break; - } + for _ in 0..150 { + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + let running = self + .supervisor + .info(id) + .await? + .is_some_and(|info| info.state.status.is_running()); + if !running { + // Synchronize Supervisor's `started` flag after the launcher + // completed its graceful child cleanup. + return self.supervisor.stop(id).await; } } - app.lock().dependency_monitors.remove(&id); - }); + warn!(id, "VM launcher did not stop gracefully; forcing shutdown"); + } + self.supervisor.stop(id).await } pub async fn remove_vm(&self, id: &str) -> Result<()> { @@ -594,10 +524,9 @@ impl App { /// `delete_workdir`: true for user-initiated removal, false for orphan cleanup. async fn finish_remove_vm(&self, id: &str, delete_workdir: bool) -> Result<()> { // Stop the supervisor process (idempotent if already stopped) - if let Err(err) = self.supervisor.stop(id).await { - debug!("supervisor.stop({id}) during removal: {err:?}"); + if let Err(err) = self.stop_vm_process(id).await { + debug!("graceful VM stop during removal failed: {err:?}"); } - self.stop_dependent_processes(id).await; // Poll until the process is no longer running, then remove it. // Some VMs take a long time to stop (e.g. 2+ hours), so we wait indefinitely. @@ -632,31 +561,6 @@ impl App { } } - // Auxiliary processes (currently swtpm) are owned by the VM through - // ProcessAnnotation::live_for. Remove their stopped supervisor entries - // before deleting the VM workdir. - for process in self.supervisor.list().await.unwrap_or_default() { - let note = - serde_json::from_str::(&process.config.note).unwrap_or_default(); - if note.live_for.as_deref() != Some(id) { - continue; - } - for _ in 0..50 { - match self.supervisor.info(&process.config.id).await { - Ok(Some(info)) if info.state.status.is_running() => { - tokio::time::sleep(std::time::Duration::from_millis(100)).await; - } - _ => break, - } - } - if let Err(err) = self.supervisor.remove(&process.config.id).await { - warn!( - process_id = process.config.id, - "failed to remove dependent process: {err:?}" - ); - } - } - // Only delete the workdir for user-initiated removal or if .removing marker exists. // Orphaned supervisor processes without the marker keep their data intact. let vm_path = self.work_dir(id); @@ -896,24 +800,16 @@ impl App { // Clean up orphaned supervisor processes (in supervisor but not loaded as VMs) let loaded_vm_ids: HashSet = self.lock().vms.keys().cloned().collect(); - for (note, process) in &running_vms { - let owner = note.live_for.as_deref().unwrap_or(&process.config.id); - if !loaded_vm_ids.contains(owner) { + for (_, process) in &running_vms { + if !loaded_vm_ids.contains(&process.config.id) { info!( "Cleaning up orphaned supervisor process: {}", process.config.id ); - self.spawn_finish_remove(owner); + self.spawn_finish_remove(&process.config.id); } } - // Re-establish lifecycle monitoring after a VMM restart. The monitor - // also reconciles a QEMU or dependent process that exited while VMM was - // unavailable. - for id in self.lock().vms.keys().cloned().collect::>() { - self.spawn_dependency_monitor(&id); - } - // Restore port forwarding for running bridge-mode VMs with persisted guest IPs let vm_ids: Vec = self.lock().vms.keys().cloned().collect(); for id in vm_ids { @@ -1705,24 +1601,6 @@ mod tests { hex::encode(vec![byte; len]) } - #[test] - fn dependency_lifecycle_is_fail_closed() { - assert_eq!( - dependency_action(true, &[true]), - DependencyAction::KeepRunning - ); - assert_eq!( - dependency_action(false, &[true]), - DependencyAction::StopDependents - ); - assert_eq!(dependency_action(true, &[false]), DependencyAction::StopVm); - assert_eq!( - dependency_action(false, &[false]), - DependencyAction::StopDependents - ); - assert_eq!(dependency_action(true, &[]), DependencyAction::Finished); - } - #[test] fn gpu_config_has_gpus_only_when_resolved_gpu_list_is_non_empty() { assert!(!GpuConfig::default().has_gpus()); @@ -2324,8 +2202,6 @@ pub(crate) struct AppState { vms: HashMap, /// Tracks active port forwarding rules per VM ID (bridge mode only). active_forwards: HashMap>, - /// VM IDs with an active auxiliary-process lifecycle monitor. - dependency_monitors: HashSet, } impl AppState { diff --git a/dstack/vmm/src/app/qemu.rs b/dstack/vmm/src/app/qemu.rs index acbdb184c..ba60703ea 100644 --- a/dstack/vmm/src/app/qemu.rs +++ b/dstack/vmm/src/app/qemu.rs @@ -6,6 +6,7 @@ use crate::{ app::Manifest, config::{CvmConfig, Networking, NetworkingMode, ProcessAnnotation, TeePlatform}, + vm_launcher::{ChildCommand, LaunchSpec}, }; use std::collections::HashMap; use std::os::unix::fs::PermissionsExt; @@ -330,7 +331,7 @@ impl VmConfig { gpus: &GpuConfig, ) -> Result> { let prepared = PreparedQemuLaunch::prepare(self, workdir, cfg, gpus)?; - let mut process = QemuCommandBuilder { + let process = QemuCommandBuilder { vm: self, cfg, gpus, @@ -345,24 +346,6 @@ impl VmConfig { .as_ref() .context("missing swtpm executable for configured socket")?; - // The supervisor starts process configs in order, but swtpm may need a - // moment to create its socket. Make QEMU wait without shell-quoting any - // of its arguments: they are passed as positional parameters. - let original_command = std::mem::replace(&mut process.command, "/bin/sh".into()); - let original_args = std::mem::take(&mut process.args); - process.args = vec![ - "-c".into(), - "for i in $(seq 1 100); do [ -S \"$1\" ] && shift && exec \"$@\"; sleep 0.05; done; echo 'timed out waiting for swtpm socket' >&2; exit 1".into(), - "dstack-qemu-wait-swtpm".into(), - socket.to_string_lossy().into_owned(), - original_command, - ]; - process.args.extend(original_args); - - let note = serde_json::to_string(&ProcessAnnotation { - kind: "swtpm".into(), - live_for: Some(self.manifest.id.clone()), - })?; let swtpm_args = vec![ "socket".into(), "--tpm2".into(), @@ -373,33 +356,42 @@ impl VmConfig { "--flags".into(), "not-need-init,startup-clear".into(), ]; - let swtpm = ProcessConfig { - id: format!("{}-swtpm", self.manifest.id), - name: format!("{} swtpm", self.manifest.name), - command: swtpm_path.to_string_lossy().into_owned(), - args: swtpm_args, - env: Default::default(), - cwd: prepared.workdir.path().to_string_lossy().into_owned(), - stdout: prepared - .workdir - .stdout_file() - .to_string_lossy() - .into_owned(), - stderr: prepared - .workdir - .stderr_file() - .to_string_lossy() - .into_owned(), - pidfile: prepared - .workdir - .swtpm_state_dir() - .join("supervisor.pid") - .to_string_lossy() - .into_owned(), - cid: None, - note, + let spec = LaunchSpec { + qemu: ChildCommand { + command: process.command, + args: process.args, + }, + swtpm: ChildCommand { + command: swtpm_path.to_string_lossy().into_owned(), + args: swtpm_args, + }, + swtpm_socket: socket.to_path_buf(), + startup_timeout_ms: 5_000, + shutdown_timeout_ms: 10_000, + }; + let spec_path = prepared.workdir.launch_spec_path(); + safe_write::safe_write(&spec_path, serde_json::to_vec_pretty(&spec)?) + .context("failed to write VM launch specification")?; + let executable = + std::env::current_exe().context("failed to locate dstack-vmm executable")?; + let launcher = ProcessConfig { + id: self.manifest.id.clone(), + name: self.manifest.name.clone(), + command: executable.to_string_lossy().into_owned(), + args: vec![ + "vm-launcher".into(), + "--spec".into(), + spec_path.to_string_lossy().into_owned(), + ], + env: process.env, + cwd: process.cwd, + stdout: process.stdout, + stderr: process.stderr, + pidfile: process.pidfile, + cid: process.cid, + note: process.note, }; - Ok(vec![swtpm, process]) + Ok(vec![launcher]) } } diff --git a/dstack/vmm/src/app/workdir.rs b/dstack/vmm/src/app/workdir.rs index b3ce4f613..7fe039f72 100644 --- a/dstack/vmm/src/app/workdir.rs +++ b/dstack/vmm/src/app/workdir.rs @@ -112,6 +112,10 @@ impl VmWorkDir { self.swtpm_state_dir().join("swtpm.sock") } + pub fn launch_spec_path(&self) -> PathBuf { + self.workdir.join("launch.json") + } + pub fn app_compose_path(&self) -> PathBuf { self.shared_dir().join(APP_COMPOSE) } diff --git a/dstack/vmm/src/main.rs b/dstack/vmm/src/main.rs index 8ab3be383..f2302dc20 100644 --- a/dstack/vmm/src/main.rs +++ b/dstack/vmm/src/main.rs @@ -30,6 +30,7 @@ mod main_routes; mod main_service; mod one_shot; mod openapi; +mod vm_launcher; const CARGO_PKG_VERSION: &str = env!("CARGO_PKG_VERSION"); const GIT_REV: &str = git_version::git_version!( @@ -60,6 +61,9 @@ enum Command { Serve, /// One-shot VM execution mode for debugging Run(RunArgs), + /// Internal per-VM QEMU/swtpm launcher. + #[command(hide = true)] + VmLauncher(VmLauncherArgs), } #[derive(ClapArgs)] @@ -74,6 +78,13 @@ struct RunArgs { dry_run: bool, } +#[derive(ClapArgs)] +struct VmLauncherArgs { + /// Path to the generated VM launch specification. + #[arg(long)] + spec: String, +} + async fn run_external_api(app: App, figment: Figment, api_auth: ApiToken) -> Result<()> { let version = app_version(); let openapi_doc = openapi::build_openapi_doc(&version)?; @@ -155,6 +166,12 @@ async fn main() -> Result<()> { let args = Args::parse(); + // The per-VM launcher must stay minimal: do not load or validate the VMM + // server configuration in this mode. + if let Some(Command::VmLauncher(launcher_args)) = &args.command { + return vm_launcher::run(Path::new(&launcher_args.spec)).await; + } + let figment = config::load_config_figment(args.config.as_deref()); let config = Config::extract_or_default(&figment)?.abs_path()?; @@ -166,6 +183,7 @@ async fn main() -> Result<()> { // Handle commands match args.command.unwrap_or_default() { + Command::VmLauncher(_) => unreachable!("launcher mode handled before config loading"), Command::Run(run_args) => { // One-shot VM execution mode return one_shot::run_one_shot( diff --git a/dstack/vmm/src/vm_launcher.rs b/dstack/vmm/src/vm_launcher.rs new file mode 100644 index 000000000..645538963 --- /dev/null +++ b/dstack/vmm/src/vm_launcher.rs @@ -0,0 +1,310 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +use std::os::unix::process::CommandExt; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use anyhow::{bail, Context, Result}; +use serde::{Deserialize, Serialize}; +use tokio::process::{Child, Command}; +use tokio::signal::unix::{signal, SignalKind}; +use tokio::time::{sleep, timeout, Instant}; +use tracing::{error, info, warn}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ChildCommand { + pub command: String, + #[serde(default)] + pub args: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct LaunchSpec { + pub qemu: ChildCommand, + pub swtpm: ChildCommand, + pub swtpm_socket: PathBuf, + #[serde(default = "default_startup_timeout_ms")] + pub startup_timeout_ms: u64, + #[serde(default = "default_shutdown_timeout_ms")] + pub shutdown_timeout_ms: u64, +} + +fn default_startup_timeout_ms() -> u64 { + 5_000 +} + +fn default_shutdown_timeout_ms() -> u64 { + 10_000 +} + +struct SocketCleanup(PathBuf); + +impl Drop for SocketCleanup { + fn drop(&mut self) { + if self.0.exists() { + if let Err(error) = fs_err::remove_file(&self.0) { + warn!(path = %self.0.display(), %error, "failed to clean up swtpm socket"); + } + } + } +} + +fn set_process_name() { + let name = b"dstack-cvm\0"; + // SAFETY: PR_SET_NAME reads a NUL-terminated buffer of at most 16 bytes. + unsafe { + libc::prctl(libc::PR_SET_NAME, name.as_ptr().cast::()); + } +} + +fn spawn_child(spec: &ChildCommand) -> Result { + let parent = unsafe { libc::getpid() }; + let mut command = Command::new(&spec.command); + command.args(&spec.args); + // SAFETY: pre_exec only invokes async-signal-safe libc operations. Checking + // the parent after PR_SET_PDEATHSIG closes the fork/parent-exit race. + unsafe { + command.as_std_mut().pre_exec(move || { + if libc::setpgid(0, 0) != 0 { + return Err(std::io::Error::last_os_error()); + } + if libc::prctl(libc::PR_SET_PDEATHSIG, libc::SIGKILL) != 0 { + return Err(std::io::Error::last_os_error()); + } + if libc::getppid() != parent { + libc::raise(libc::SIGKILL); + } + Ok(()) + }); + } + command + .spawn() + .with_context(|| format!("failed to start {}", spec.command)) +} + +async fn stop_child(child: &mut Child, name: &str, grace: Duration) { + let Some(pid) = child.id() else { + return; + }; + // SAFETY: pid comes from the live Child handle and SIGTERM has no pointer arguments. + if unsafe { libc::kill(-(pid as libc::pid_t), libc::SIGTERM) } != 0 { + let error = std::io::Error::last_os_error(); + if error.raw_os_error() != Some(libc::ESRCH) { + warn!(%pid, %name, %error, "failed to terminate child"); + } + } + match timeout(grace, child.wait()).await { + Ok(Ok(status)) => info!(%pid, %name, %status, "child stopped"), + Ok(Err(error)) => warn!(%pid, %name, %error, "failed to wait for child"), + Err(_) => { + warn!(%pid, %name, "child did not stop gracefully; killing"); + if unsafe { libc::kill(-(pid as libc::pid_t), libc::SIGKILL) } != 0 { + let error = std::io::Error::last_os_error(); + if error.raw_os_error() != Some(libc::ESRCH) { + warn!(%pid, %name, %error, "failed to kill child process group"); + } + } + if let Err(error) = child.wait().await { + warn!(%pid, %name, %error, "failed to reap killed child"); + } + } + } + // The group normally disappears with its foreground process. Kill any + // helper descendants that outlived the group leader before launcher exits. + if unsafe { libc::kill(-(pid as libc::pid_t), libc::SIGKILL) } != 0 { + let error = std::io::Error::last_os_error(); + if error.raw_os_error() != Some(libc::ESRCH) { + warn!(%pid, %name, %error, "failed to clean up child process group"); + } + } +} + +async fn wait_for_swtpm(swtpm: &mut Child, socket: &Path, deadline: Instant) -> Result<()> { + loop { + if socket.exists() { + return Ok(()); + } + if Instant::now() >= deadline { + bail!("timed out waiting for swtpm socket {}", socket.display()); + } + tokio::select! { + status = swtpm.wait() => { + bail!("swtpm exited before socket was ready: {}", status?); + } + _ = sleep(Duration::from_millis(50)) => {} + } + } +} + +pub async fn run(spec_path: &Path) -> Result<()> { + set_process_name(); + let raw = fs_err::read(spec_path) + .with_context(|| format!("failed to read launch spec {}", spec_path.display()))?; + let spec: LaunchSpec = serde_json::from_slice(&raw).context("failed to parse launch spec")?; + let _socket_cleanup = SocketCleanup(spec.swtpm_socket.clone()); + if spec.swtpm_socket.exists() { + fs_err::remove_file(&spec.swtpm_socket).context("failed to remove stale swtpm socket")?; + } + + let grace = Duration::from_millis(spec.shutdown_timeout_ms); + let mut terminate = signal(SignalKind::terminate()).context("failed to watch SIGTERM")?; + let mut interrupt = signal(SignalKind::interrupt()).context("failed to watch SIGINT")?; + let mut swtpm = spawn_child(&spec.swtpm)?; + enum StartupExit { + Ready, + Error(anyhow::Error), + Signal, + } + let startup_exit = { + let startup = wait_for_swtpm( + &mut swtpm, + &spec.swtpm_socket, + Instant::now() + Duration::from_millis(spec.startup_timeout_ms), + ); + tokio::pin!(startup); + tokio::select! { + result = &mut startup => match result { + Ok(()) => StartupExit::Ready, + Err(error) => StartupExit::Error(error), + }, + _ = terminate.recv() => StartupExit::Signal, + _ = interrupt.recv() => StartupExit::Signal, + } + }; + match startup_exit { + StartupExit::Ready => {} + StartupExit::Error(error) => { + stop_child(&mut swtpm, "swtpm", grace).await; + return Err(error); + } + StartupExit::Signal => { + stop_child(&mut swtpm, "swtpm", grace).await; + return Ok(()); + } + } + + let mut qemu = match spawn_child(&spec.qemu) { + Ok(child) => child, + Err(error) => { + stop_child(&mut swtpm, "swtpm", grace).await; + return Err(error); + } + }; + info!( + qemu_pid = qemu.id(), + swtpm_pid = swtpm.id(), + "VM processes started" + ); + + enum Exit { + Signal, + Qemu(std::process::ExitStatus), + Swtpm(std::process::ExitStatus), + } + let exit = tokio::select! { + _ = terminate.recv() => Exit::Signal, + _ = interrupt.recv() => Exit::Signal, + status = qemu.wait() => Exit::Qemu(status.context("failed to wait for QEMU")?), + status = swtpm.wait() => Exit::Swtpm(status.context("failed to wait for swtpm")?), + }; + + match exit { + Exit::Signal => { + tokio::join!( + stop_child(&mut qemu, "qemu", grace), + stop_child(&mut swtpm, "swtpm", grace) + ); + Ok(()) + } + Exit::Qemu(status) => { + stop_child(&mut swtpm, "swtpm", grace).await; + if status.success() { + Ok(()) + } else { + bail!("QEMU exited with {status}") + } + } + Exit::Swtpm(status) => { + error!(%status, "swtpm exited while QEMU was running"); + stop_child(&mut qemu, "qemu", grace).await; + bail!("swtpm exited with {status}") + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::os::unix::net::UnixListener; + + fn shell(script: String) -> ChildCommand { + ChildCommand { + command: "/bin/sh".into(), + args: vec!["-c".into(), script], + } + } + + fn process_is_gone(pid: libc::pid_t) -> bool { + // SAFETY: signal 0 only probes whether the PID still exists. + let missing = unsafe { libc::kill(pid, 0) != 0 }; + missing && std::io::Error::last_os_error().raw_os_error() == Some(libc::ESRCH) + } + + async fn create_fake_socket(path: PathBuf) { + sleep(Duration::from_millis(100)).await; + let _listener = UnixListener::bind(path).unwrap(); + sleep(Duration::from_secs(5)).await; + } + + #[tokio::test] + async fn qemu_failure_stops_and_reaps_swtpm() { + let dir = tempfile::tempdir().unwrap(); + let socket = dir.path().join("swtpm.sock"); + let swtpm_pid = dir.path().join("swtpm.pid"); + let spec = LaunchSpec { + qemu: shell("exit 7".into()), + swtpm: shell(format!("echo $$ > {}; sleep 30", swtpm_pid.display())), + swtpm_socket: socket.clone(), + startup_timeout_ms: 2_000, + shutdown_timeout_ms: 500, + }; + let spec_path = dir.path().join("spec.json"); + fs_err::write(&spec_path, serde_json::to_vec(&spec).unwrap()).unwrap(); + tokio::spawn(create_fake_socket(socket)); + + assert!(run(&spec_path).await.is_err()); + let pid: libc::pid_t = fs_err::read_to_string(swtpm_pid) + .unwrap() + .trim() + .parse() + .unwrap(); + assert!(process_is_gone(pid)); + } + + #[tokio::test] + async fn swtpm_failure_stops_and_reaps_qemu() { + let dir = tempfile::tempdir().unwrap(); + let socket = dir.path().join("swtpm.sock"); + let qemu_pid = dir.path().join("qemu.pid"); + let spec = LaunchSpec { + qemu: shell(format!("echo $$ > {}; sleep 30", qemu_pid.display())), + swtpm: shell("sleep 0.2; exit 9".into()), + swtpm_socket: socket.clone(), + startup_timeout_ms: 2_000, + shutdown_timeout_ms: 500, + }; + let spec_path = dir.path().join("spec.json"); + fs_err::write(&spec_path, serde_json::to_vec(&spec).unwrap()).unwrap(); + tokio::spawn(create_fake_socket(socket)); + + assert!(run(&spec_path).await.is_err()); + let pid: libc::pid_t = fs_err::read_to_string(qemu_pid) + .unwrap() + .trim() + .parse() + .unwrap(); + assert!(process_is_gone(pid)); + } +} From ecb6cf2f550ed90dcc394d551db80635465d8cf9 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 19 Jul 2026 21:36:44 -0700 Subject: [PATCH 05/12] fix(vmm): restrict swtpm socket to QEMU user --- dstack/vmm/src/app/qemu.rs | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/dstack/vmm/src/app/qemu.rs b/dstack/vmm/src/app/qemu.rs index ba60703ea..e74f71257 100644 --- a/dstack/vmm/src/app/qemu.rs +++ b/dstack/vmm/src/app/qemu.rs @@ -30,6 +30,7 @@ use anyhow::{bail, Context, Result}; use bon::Builder; use dstack_types::{shared_filenames::HOST_SHARED_DISK_LABEL, KeyProviderKind}; use fs_err as fs; +use nix::unistd::User; use serde::Serialize; use supervisor_client::supervisor::ProcessConfig; @@ -345,6 +346,14 @@ impl VmConfig { .swtpm_path .as_ref() .context("missing swtpm executable for configured socket")?; + let (socket_uid, socket_gid) = if cfg.user.is_empty() { + (unsafe { libc::geteuid() }, unsafe { libc::getegid() }) + } else { + let user = User::from_name(&cfg.user) + .context("failed to resolve QEMU user")? + .with_context(|| format!("QEMU user {} does not exist", cfg.user))?; + (user.uid.as_raw(), user.gid.as_raw()) + }; let swtpm_args = vec![ "socket".into(), @@ -352,7 +361,10 @@ impl VmConfig { "--tpmstate".into(), format!("dir={}", prepared.workdir.swtpm_state_dir().display()), "--ctrl".into(), - format!("type=unixio,path={},mode=0600", socket.display()), + format!( + "type=unixio,path={},mode=0600,uid={socket_uid},gid={socket_gid}", + socket.display() + ), "--flags".into(), "not-need-init,startup-clear".into(), ]; From ef07f67bb995bea9ee896d9b41d17bd3514dd17e Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 19 Jul 2026 21:46:47 -0700 Subject: [PATCH 06/12] refactor(vmm): keep launcher process name unchanged --- dstack/vmm/src/vm_launcher.rs | 9 --------- 1 file changed, 9 deletions(-) diff --git a/dstack/vmm/src/vm_launcher.rs b/dstack/vmm/src/vm_launcher.rs index 645538963..3a9267751 100644 --- a/dstack/vmm/src/vm_launcher.rs +++ b/dstack/vmm/src/vm_launcher.rs @@ -51,14 +51,6 @@ impl Drop for SocketCleanup { } } -fn set_process_name() { - let name = b"dstack-cvm\0"; - // SAFETY: PR_SET_NAME reads a NUL-terminated buffer of at most 16 bytes. - unsafe { - libc::prctl(libc::PR_SET_NAME, name.as_ptr().cast::()); - } -} - fn spawn_child(spec: &ChildCommand) -> Result { let parent = unsafe { libc::getpid() }; let mut command = Command::new(&spec.command); @@ -139,7 +131,6 @@ async fn wait_for_swtpm(swtpm: &mut Child, socket: &Path, deadline: Instant) -> } pub async fn run(spec_path: &Path) -> Result<()> { - set_process_name(); let raw = fs_err::read(spec_path) .with_context(|| format!("failed to read launch spec {}", spec_path.display()))?; let spec: LaunchSpec = serde_json::from_slice(&raw).context("failed to parse launch spec")?; From 22a6286e4bafc13b5fa72c88693abb1084ea5c6b Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 19 Jul 2026 21:49:28 -0700 Subject: [PATCH 07/12] refactor(vmm): allow swtpm for all Dstack VMs --- CONTRIBUTING.md | 2 +- docs/security/cvm-boundaries.md | 2 +- dstack/vmm/src/app/qemu.rs | 3 --- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 02edd7d2d..196b838a5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -66,7 +66,7 @@ To exercise persistent TPM-backed app keys as well, install `swtpm` on the VMM host and select `key_provider=tpm` in the VMM console (or pass `--key-provider tpm` in tooling that exposes the compose option). The VMM keeps the software TPM state in the VM work directory so that seal/unseal survives guest restarts. -This development provider is available only to no-TEE development images. +The software TPM is host-controlled and does not provide hardware isolation. The simulator package is installed only in development images. This mode provides no hardware isolation and must never be used with production diff --git a/docs/security/cvm-boundaries.md b/docs/security/cvm-boundaries.md index aea673f5f..fed4f084f 100644 --- a/docs/security/cvm-boundaries.md +++ b/docs/security/cvm-boundaries.md @@ -44,7 +44,7 @@ This is the main configuration file for the application in JSON format: | init_script | 0.5.5 | string | Bash script that executed prior to dockerd startup | | storage_fs | 0.5.5 | string | Filesystem type for the data disk of the CVM. Supported values: "zfs", "ext4". default to "zfs". **ZFS:** Ensures filesystem integrity with built-in data protection features. **ext4:** Provides better performance for database applications with lower overhead and faster I/O operations, but no strong integrity protection. | | swap_size | 0.5.5 | string/integer | The linux swap size. default to 0. Can be in byte or human-readable format (e.g., "1G", "256M"). | -| key_provider | 0.5.6 | string | Key provider type. Supported values: "none", "kms", "local", "tpm". GCP vTPM and AWS EC2 NitroTPM are part of their platform trust models. The Dstack development platform can use VMM-managed swtpm to exercise seal/unseal and restart persistence, but it offers no protection against the host and is intentionally not accepted by remote verifiers. | +| key_provider | 0.5.6 | string | Key provider type. Supported values: "none", "kms", "local", "tpm". GCP vTPM and AWS EC2 NitroTPM are part of their platform trust models. The Dstack platform can use VMM-managed swtpm for seal/unseal and restart persistence, but it offers no protection against the host and is intentionally not accepted by remote verifiers. | The hash of this file content is extended as the dstack `compose-hash` launch event. On TDX-family platforms the launch event is measured into RTMR3. On AWS NitroTPM it is measured into non-resettable SHA384 PCR14 before the `system-ready` launch boundary. Remote verifiers extract and replay this event during attestation. diff --git a/dstack/vmm/src/app/qemu.rs b/dstack/vmm/src/app/qemu.rs index e74f71257..270c62c73 100644 --- a/dstack/vmm/src/app/qemu.rs +++ b/dstack/vmm/src/app/qemu.rs @@ -224,9 +224,6 @@ impl PreparedQemuLaunch { }; let (swtpm_socket, swtpm_path) = if matches!(app_compose.key_provider(), KeyProviderKind::Tpm) { - if !vm.manifest.no_tee || !vm.image.info.is_dev { - bail!("swtpm key provider is only available to no-TEE development VMs"); - } let swtpm_path = which::which("swtpm") .context("tpm key provider requested but swtpm is not installed")?; let state_dir = workdir.swtpm_state_dir(); From 518394f226562f4216995cc5a254350c61079d09 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 19 Jul 2026 21:56:10 -0700 Subject: [PATCH 08/12] fix(vmm): preserve direct QEMU lifecycle for non-TPM VMs --- dstack/vmm/src/app.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/dstack/vmm/src/app.rs b/dstack/vmm/src/app.rs index 5c9ea15ae..bf5aa343d 100644 --- a/dstack/vmm/src/app.rs +++ b/dstack/vmm/src/app.rs @@ -462,6 +462,12 @@ impl App { let Some(info) = self.supervisor.info(id).await? else { return Ok(()); }; + // Non-TPM VMs run QEMU directly and keep the existing Supervisor stop + // path. Only the TPM launcher's hidden subcommand implements graceful + // child-process shutdown. + if info.config.args.first().map(String::as_str) != Some("vm-launcher") { + return self.supervisor.stop(id).await; + } if info.state.status.is_running() { let pid = info.state.pid.context("running VM launcher has no PID")?; if let Err(error) = signal_pidfd(pid, libc::SIGTERM) { From 51d7528ff84cf280bb0ade2af690ac657e72bf35 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 19 Jul 2026 22:13:58 -0700 Subject: [PATCH 09/12] refactor(vmm): remove unsupported v0 console --- dstack/vmm/src/console_v0.html | 2632 ----------------- dstack/vmm/src/main_routes.rs | 7 +- .../vmm/ui/src/components/CreateVmDialog.ts | 2 +- dstack/vmm/ui/src/composables/useVmManager.ts | 4 +- 4 files changed, 5 insertions(+), 2640 deletions(-) delete mode 100644 dstack/vmm/src/console_v0.html diff --git a/dstack/vmm/src/console_v0.html b/dstack/vmm/src/console_v0.html deleted file mode 100644 index c814a9432..000000000 --- a/dstack/vmm/src/console_v0.html +++ /dev/null @@ -1,2632 +0,0 @@ - - - - - - - - - {{TITLE}} - - - - - - - - - - - - - - - - - - - - -
- - -
-
-

Deploy a new instance

-
-
-
- - -
- -
- - -
- -
- - -
- -
- -
- - -
-
- -
- -
- - -
- Leave as 0 to disable swap. -
- -
- - -
- -
- - -
- -
- -
-
- - or paste below - -
- -
-
-
- -
- - - -
- -
- - -
- -
- - -
- -
- - -
- -
- -
- -
- -
- - - - - - - -
-
-
- - -
- -
- -
- -
- -
-
- - -
-
- - -
-
-
-
- Compose Hash: 0x{{ composeHashPreview }} -
- -
- - -
-
-
-
- -

VM List

-
-
-
- - -
-
- Total: {{ totalVMs }} -
-
- - - - - - - - - - - - - -
NameStatusUptimeViewActions
-
- - -
- Page - - of {{ maxPage || 1 }} -
- - - - - -
-
- -
-
-

Update VM Config

- -
-
- - -
-
- -
- - -
-
-
- -
- -
- - -
- Enable "Update compose" to change swap size. -
- -
- - -
- -
-
- - -
- - -
-
- -
- -
- - -
-
-
- -
-
-
- -
-
- - or paste below - -
- -
-
-
- - -
-
- Compose Hash: 0x{{ upgradeComposeHashPreview }} -
-
-
- -
-
- -
-
- -
- -
- -
- - -
- -
- - -
-
-
- -
-
-

Derive VM

-

- This will create a new VM instance with the same app id, but the disk state will NOT migrate to the - new instance. -

- -
- - -
- -
- - -
-
- - -
-
- - -
-
- - -
-
-
- -
-
- -
-
-
- {{ errorMessage }} - -
-
-
- Version: v{{ version.version }} ({{ version.rev }}) -
-
- - - - - - - - - - - - - diff --git a/dstack/vmm/src/main_routes.rs b/dstack/vmm/src/main_routes.rs index c74e7e63b..13fcffa57 100644 --- a/dstack/vmm/src/main_routes.rs +++ b/dstack/vmm/src/main_routes.rs @@ -57,11 +57,6 @@ async fn beta(app: &State) -> (ContentType, String) { index(app).await } -#[get("/v0")] -async fn v0(app: &State) -> (ContentType, String) { - render_console(file_or_include_str!("console_v0.html"), app) -} - #[get("/res/")] async fn res(path: &str) -> Result<(ContentType, String), Custom> { match path { @@ -183,5 +178,5 @@ fn vm_logs( } pub fn routes() -> Vec { - routes![index, v1, beta, v0, res, vm_logs] + routes![index, v1, beta, res, vm_logs] } diff --git a/dstack/vmm/ui/src/components/CreateVmDialog.ts b/dstack/vmm/ui/src/components/CreateVmDialog.ts index c0e59eb20..af7dd45fb 100644 --- a/dstack/vmm/ui/src/components/CreateVmDialog.ts +++ b/dstack/vmm/ui/src/components/CreateVmDialog.ts @@ -145,7 +145,7 @@ const CreateVmDialogComponent = {
-
+
diff --git a/dstack/vmm/ui/src/composables/useVmManager.ts b/dstack/vmm/ui/src/composables/useVmManager.ts index c6c60c741..ff5135a52 100644 --- a/dstack/vmm/ui/src/composables/useVmManager.ts +++ b/dstack/vmm/ui/src/composables/useVmManager.ts @@ -813,7 +813,9 @@ type CreateVmPayloadSource = { public_logs: vmForm.value.public_logs, public_sysinfo: vmForm.value.public_sysinfo, public_tcbinfo: vmForm.value.public_tcbinfo, - key_provider_id: vmForm.value.key_provider_id, + key_provider_id: vmForm.value.key_provider === 'kms' || vmForm.value.key_provider === 'local' + ? vmForm.value.key_provider_id + : '', allowed_envs: vmForm.value.encryptedEnvs.map((env) => env.key), no_instance_id: !vmForm.value.gateway_enabled, secure_time: false, From c8c259d6ae69bf55e57bec00fe600d459ca36209 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 19 Jul 2026 22:16:47 -0700 Subject: [PATCH 10/12] fix(vmm-ui): expose swtpm and no-TEE controls --- dstack/vmm/ui/src/components/CreateVmDialog.ts | 4 ++-- dstack/vmm/ui/src/composables/useVmManager.ts | 6 ------ dstack/vmm/ui/src/templates/app.html | 7 ------- 3 files changed, 2 insertions(+), 15 deletions(-) diff --git a/dstack/vmm/ui/src/components/CreateVmDialog.ts b/dstack/vmm/ui/src/components/CreateVmDialog.ts index af7dd45fb..0eabb9cd3 100644 --- a/dstack/vmm/ui/src/components/CreateVmDialog.ts +++ b/dstack/vmm/ui/src/components/CreateVmDialog.ts @@ -141,7 +141,7 @@ const CreateVmDialogComponent = { - +
@@ -184,7 +184,7 @@ const CreateVmDialogComponent = { - +
diff --git a/dstack/vmm/ui/src/composables/useVmManager.ts b/dstack/vmm/ui/src/composables/useVmManager.ts index ff5135a52..609de8aa5 100644 --- a/dstack/vmm/ui/src/composables/useVmManager.ts +++ b/dstack/vmm/ui/src/composables/useVmManager.ts @@ -1362,11 +1362,6 @@ type CreateVmPayloadSource = { window.open('/api-docs/docs', '_blank', 'noopener'); } - function openLegacyUi() { - closeSystemMenu(); - window.open('/v0', '_blank', 'noopener'); - } - function shortUptime(uptime?: string | null) { if (!uptime) { return '-'; @@ -1809,7 +1804,6 @@ type CreateVmPayloadSource = { toggleSystemMenu, closeSystemMenu, openApiDocs, - openLegacyUi, reloadVMs, devMode, toggleDevMode, diff --git a/dstack/vmm/ui/src/templates/app.html b/dstack/vmm/ui/src/templates/app.html index 7e38806c7..139155032 100644 --- a/dstack/vmm/ui/src/templates/app.html +++ b/dstack/vmm/ui/src/templates/app.html @@ -57,13 +57,6 @@

dstack-vmm

API Docs -