feat(vmm): add optional libvirt network filtering

docs/libvirt-network-filter.md

@@ -0,0 +1,111 @@
+# Optional libvirt network filtering
+
+## Goal
+
+Allow bridge-backed VMs to opt into an existing libvirt `nwfilter` without
+allowing libvirt to create or launch the QEMU domain. QEMU remains entirely
+owned by `dstack-vmm`, so its command line and attestation inputs do not change
+outside the explicitly selected network backend.
+
+The measurable acceptance criteria are:
+
+- `network_filter = "none"` preserves the existing QEMU `-netdev bridge`
+  behavior and does not require `netd` or libvirt.
+- `network_filter = "libvirt"` creates the TAP and filter binding before QEMU
+  is submitted to Supervisor, and uses QEMU `-netdev tap`.
+- A failed TAP or filter setup prevents QEMU from starting and rolls back all
+  interfaces prepared for that VM.
+- Normal stop and removal delete the filter binding and TAP.
+- One host `netd` can serve multiple VMM instances. Resource names include a
+  stable VMM instance namespace, VM ID, and NIC index.
+- Development builds can run the VMM and `netd` directly, with explicit socket
+  and allowed-UID command-line options; systemd is not required.
+
+## Configuration
+
+Filtering is a VMM host policy, not a field accepted from a VM manifest:
+
+```toml
+[cvm.network_filter]
+mode = "none" # or "libvirt"
+filter = "clean-traffic"
+parameters = {}
+
+[netd]
+socket = "/run/dstack/netd.sock"
+allowed_uids = [] # empty means root only
+libvirt_uri = "qemu:///system"
+```
+
+Each VMM instance also has an `instance_id`. It must be unique among VMMs that
+share a host. If omitted, the VMM derives a stable namespace from its absolute
+run directory.
+
+## Architecture
+
+`netd` is a host-level privilege broker. It accepts a small, bounded JSON
+protocol over a Unix stream socket and authorizes clients with `SO_PEERCRED`.
+A single process can serve multiple VMMs; a dedicated process can use another
+socket for development or isolation.
+
+For libvirt mode, startup is:
+
+1. Derive the TAP name from instance namespace, VM ID, and NIC index.
+2. Create the TAP for the configured QEMU UID and attach it to the bridge.
+3. Create a libvirt nwfilter binding for the TAP.
+4. Bring the TAP up and return success.
+5. Start QEMU directly with `-netdev tap,script=no,downscript=no`.
+
+Teardown stops QEMU first, removes the binding, and deletes the TAP. Operations
+are serialized by `netd`. The design intentionally does not add ownership
+aliases; deployments must use unique instance IDs.
+
+`netd` invokes fixed absolute `ip` and `virsh` executables with separate
+arguments. It never accepts a command, executable path, TAP name, or raw XML
+from a client. Filter XML is generated internally with XML escaping and is
+validated by libvirt.
+
+## Deployment modes
+
+Production should run one shared service. `netd` reads only the `[netd]`
+section, so its root-owned configuration can be small and independent of every
+VMM instance:
+
+```toml
+# /etc/dstack/netd.toml
+[netd]
+socket = "/run/dstack/netd.sock"
+allowed_uids = [991, 992]
+libvirt_uri = "qemu:///system"
+```
+
+```ini
+# /etc/systemd/system/dstack-netd.service
+[Unit]
+Description=dstack host networking service
+After=libvirtd.service
+
+[Service]
+ExecStart=/usr/bin/dstack-vmm --config /etc/dstack/netd.toml netd
+Restart=on-failure
+
+[Install]
+WantedBy=multi-user.target
+```
+
+All VMM instance configurations point to the same socket and use distinct
+`cvm.instance_id` values. A dedicated netd uses a different socket. A
+host-wide lock serializes mutations made by shared and dedicated netd
+processes.
+
+Development mode is two ordinary commands:
+
+```bash
+sudo dstack-vmm --config ./vmm.toml netd \
+  --socket /run/dstack-dev/netd.sock --allow-uid "$(id -u)"
+dstack-vmm --config ./vmm.toml \
+  --netd-socket /run/dstack-dev/netd.sock
+```
+
+User networking and bridge networking with `mode = "none"` never connect to
+`netd`. Libvirt mode fails closed if `netd` is unavailable.

dstack/vmm/src/app.rs

@@ -2,7 +2,10 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 
-use crate::config::{Config, Networking, ProcessAnnotation, Protocol};
+use crate::{
+    config::{Config, NetworkFilterMode, Networking, NetworkingMode, ProcessAnnotation, Protocol},
+    netd::{self, InterfaceIdentity, PrepareRequest, Request as NetdRequest},
+};
 
 use anyhow::{bail, Context, Result};
 use bon::Builder;
@@ -17,6 +20,7 @@ use dstack_vmm_rpc::{
 use fs_err as fs;
 use guest_api::client::DefaultClient as GuestClient;
 use id_pool::IdPool;
+use nix::unistd::User;
 use or_panic::ResultOrPanic;
 use ra_rpc::client::RaClient;
 use serde::{Deserialize, Serialize};
@@ -423,17 +427,30 @@ impl App {
             append_boot_separator(&work_dir.stdout_file());
             append_boot_separator(&work_dir.stderr_file());
 
+            let runtime_networks = resolved_networks(&vm_config.manifest, &self.config.cvm);
             let devices = self.try_allocate_gpus(&vm_config.manifest)?;
             let processes = vm_config.config_qemu(&work_dir, &self.config.cvm, &devices)?;
-            let runtime_networks = resolved_networks(&vm_config.manifest, &self.config.cvm);
             work_dir.set_runtime_networks(&runtime_networks)?;
+            if let Err(error) = self
+                .prepare_filtered_networks(&vm_config, &runtime_networks)
+                .await
+            {
+                let _ = work_dir.clear_runtime_networks();
+                return Err(error);
+            }
             {
                 let mut state = self.lock();
                 let vm_state = state.get_mut(id).context("VM not found")?;
-                vm_state.state.runtime_networks = runtime_networks;
+                vm_state.state.runtime_networks = runtime_networks.clone();
             }
             for process in processes {
                 if let Err(err) = self.supervisor.deploy(&process).await {
+                    if let Err(cleanup_error) = self
+                        .remove_filtered_networks(&vm_config.manifest.id, &runtime_networks)
+                        .await
+                    {
+                        warn!(id, %cleanup_error, "failed to roll back filtered networking");
+                    }
                     if let Err(clear_err) = work_dir.clear_runtime_networks() {
                         warn!(
                             id,
@@ -465,6 +482,88 @@ impl App {
     pub async fn stop_vm(&self, id: &str) -> Result<()> {
         self.set_started(id, false)?;
         self.stop_vm_process(id).await?;
+        let networks = self.work_dir(id).runtime_networks();
+        self.remove_filtered_networks(id, &networks).await?;
+        Ok(())
+    }
+
+    async fn prepare_filtered_networks(
+        &self,
+        vm: &VmConfig,
+        networks: &[Networking],
+    ) -> Result<()> {
+        if self.config.cvm.network_filter.mode == NetworkFilterMode::None {
+            return Ok(());
+        }
+        let qemu_uid = if self.config.cvm.user.is_empty() {
+            unsafe { libc::geteuid() }
+        } else {
+            User::from_name(&self.config.cvm.user)
+                .context("failed to resolve QEMU user")?
+                .with_context(|| format!("QEMU user {} does not exist", self.config.cvm.user))?
+                .uid
+                .as_raw()
+        };
+        let mut prepared = Vec::new();
+        for (nic_index, network) in networks.iter().enumerate() {
+            if network.mode != NetworkingMode::Bridge {
+                continue;
+            }
+            let identity = InterfaceIdentity {
+                instance_id: self.config.cvm.instance_id.clone(),
+                vm_id: vm.manifest.id.clone(),
+                nic_index,
+            };
+            let request = PrepareRequest {
+                identity: identity.clone(),
+                bridge: network.bridge.clone(),
+                mac: network::mac_address_for_vm_index(
+                    &vm.manifest.id,
+                    &network.mac_prefix_bytes(),
+                    nic_index,
+                ),
+                qemu_uid,
+                filter: self.config.cvm.network_filter.filter.clone(),
+                parameters: self.config.cvm.network_filter.parameters.clone(),
+            };
+            if let Err(error) =
+                netd::request(&self.config.netd.socket, &NetdRequest::Prepare(request)).await
+            {
+                for identity in prepared.into_iter().rev() {
+                    let _ =
+                        netd::request(&self.config.netd.socket, &NetdRequest::Remove { identity })
+                            .await;
+                }
+                return Err(error).context("failed to prepare libvirt-filtered networking");
+            }
+            prepared.push(identity);
+        }
+        Ok(())
+    }
+
+    async fn remove_filtered_networks(&self, vm_id: &str, networks: &[Networking]) -> Result<()> {
+        if self.config.cvm.network_filter.mode == NetworkFilterMode::None {
+            return Ok(());
+        }
+        let mut first_error = None;
+        for (nic_index, network) in networks.iter().enumerate().rev() {
+            if network.mode != NetworkingMode::Bridge {
+                continue;
+            }
+            let identity = InterfaceIdentity {
+                instance_id: self.config.cvm.instance_id.clone(),
+                vm_id: vm_id.to_string(),
+                nic_index,
+            };
+            if let Err(error) =
+                netd::request(&self.config.netd.socket, &NetdRequest::Remove { identity }).await
+            {
+                first_error.get_or_insert(error);
+            }
+        }
+        if let Some(error) = first_error {
+            return Err(error).context("failed to remove libvirt-filtered networking");
+        }
         Ok(())
     }
 
@@ -574,6 +673,11 @@ impl App {
             }
         }
 
+        let runtime_networks = self.work_dir(id).runtime_networks();
+        if let Err(error) = self.remove_filtered_networks(id, &runtime_networks).await {
+            warn!(id, %error, "failed to remove filtered networking during VM removal");
+        }
+
         // Only delete the workdir for user-initiated removal or if .removing marker exists.
         // Orphaned supervisor processes without the marker keep their data intact.
         let vm_path = self.work_dir(id);

dstack/vmm/src/app/qemu.rs

@@ -14,7 +14,10 @@ use super::{
 };
 use crate::{
     app::Manifest,
-    config::{CvmConfig, CvmPlatform, Networking, NetworkingMode, ProcessAnnotation},
+    config::{
+        CvmConfig, CvmPlatform, NetworkFilterMode, Networking, NetworkingMode, ProcessAnnotation,
+    },
+    netd::{tap_name, InterfaceIdentity},
     vm_launcher::{ChildCommand, LaunchSpec},
 };
 use anyhow::{bail, Context, Result};
@@ -600,7 +603,21 @@ impl QemuCommandBuilder<'_> {
                 }
                 NetworkingMode::Bridge => {
                     tracing::info!("bridge networking: mac={mac} bridge={}", networking.bridge);
-                    format!("bridge,id={net_id},br={}", networking.bridge)
+                    match self.cfg.network_filter.mode {
+                        NetworkFilterMode::None => {
+                            format!("bridge,id={net_id},br={}", networking.bridge)
+                        }
+                        NetworkFilterMode::Libvirt => {
+                            let tap = tap_name(&InterfaceIdentity {
+                                instance_id: self.cfg.instance_id.clone(),
+                                vm_id: self.vm.manifest.id.clone(),
+                                nic_index: index,
+                            });
+                            format!(
+                                "tap,id={net_id},ifname={tap},script=no,downscript=no,vhost=off"
+                            )
+                        }
+                    }
                 }
                 NetworkingMode::Custom => {
                     if !networking.netdev.contains(&format!("id={net_id}")) {
@@ -976,7 +993,10 @@ mod tests {
     };
     use crate::app::image::{Image, ImageInfo};
     use crate::app::{needs_swtpm, GpuConfig, Manifest, PortMapping, VmVolume, VmWorkDir};
-    use crate::config::{Config, CvmPlatform, Protocol, DEFAULT_CONFIG};
+    use crate::config::{
+        Config, CvmPlatform, NetworkFilterMode, NetworkingMode, Protocol, DEFAULT_CONFIG,
+    };
+    use crate::netd::{tap_name, InterfaceIdentity};
     use dstack_types::{KeyProviderKind, TeeVariant};
 
     #[test]
@@ -1173,6 +1193,42 @@ mod tests {
             .iter()
             .any(|arg| arg.contains("virtio-net-pci,netdev=net1")));
 
+        for network in &mut prepared.networks {
+            network.mode = NetworkingMode::Bridge;
+            network.bridge = "br0".into();
+        }
+        let process = QemuCommandBuilder {
+            vm: &vm,
+            cfg: &config.cvm,
+            gpus: &GpuConfig::default(),
+            prepared: &prepared,
+        }
+        .build()
+        .unwrap();
+        assert!(process
+            .args
+            .iter()
+            .any(|arg| arg == "bridge,id=net0,br=br0"));
+
+        config.cvm.instance_id = "vmm-a".into();
+        config.cvm.network_filter.mode = NetworkFilterMode::Libvirt;
+        let process = QemuCommandBuilder {
+            vm: &vm,
+            cfg: &config.cvm,
+            gpus: &GpuConfig::default(),
+            prepared: &prepared,
+        }
+        .build()
+        .unwrap();
+        let expected_tap = tap_name(&InterfaceIdentity {
+            instance_id: "vmm-a".into(),
+            vm_id: "vm-1".into(),
+            nic_index: 0,
+        });
+        assert!(process.args.iter().any(|arg| {
+            arg == &format!("tap,id=net0,ifname={expected_tap},script=no,downscript=no,vhost=off")
+        }));
+
         prepared.swtpm_socket = Some(PathBuf::from("/does-not-exist/vm-1/swtpm/swtpm.sock"));
         let process = QemuCommandBuilder {
             vm: &vm,