dragonball: Support vhost-user-net devices

This PR introduces vhost-user-net devices to Dragonball. The devices are
allowed to run as server on the VMM side.

Fixes: #8502

Signed-off-by: Eric Ren <renzhen@linux.alibaba.com>
Signed-off-by: Liu Jiang <gerry@linux.alibaba.com>
Signed-off-by: Zha Bin <zhabin@linux.alibaba.com>
Signed-off-by: Chao Wu <chaowu@linux.alibaba.com>
Signed-off-by: Zizheng Bian <zizheng.bian@linux.alibaba.com>
Signed-off-by: Xuewei Niu <niuxuewei.nxw@antgroup.com>

src/dragonball/Cargo.toml

@@ -64,4 +64,5 @@ virtio-fs = ["dbs-virtio-devices/virtio-fs-pro", "virtio-queue", "atomic-guest-m
 virtio-mem = ["dbs-virtio-devices/virtio-mem", "virtio-queue", "atomic-guest-memory"]
 virtio-balloon = ["dbs-virtio-devices/virtio-balloon", "virtio-queue"]
 vhost-net = ["dbs-virtio-devices/vhost-net"]
-vhost-user-fs = ["dbs-virtio-devices/vhost-user-fs"]
+vhost-user-fs = ["dbs-virtio-devices/vhost-user-fs"]
+vhost-user-net = ["dbs-virtio-devices/vhost-user-net"]

src/dragonball/src/api/v1/mod.rs

@@ -19,7 +19,17 @@ mod machine_config;
 pub use self::machine_config::{VmConfigError, MAX_SUPPORTED_VCPUS};
 
 /// Wrapper for configuring the virtio networking
-#[cfg(any(feature = "virtio-net", feature = "vhost-net"))]
+#[cfg(any(
+    feature = "virtio-net",
+    feature = "vhost-net",
+    feature = "vhost-user-net"
+))]
 mod virtio_net;
 #[cfg(any(feature = "virtio-net", feature = "vhost-net"))]
-pub use virtio_net::{Backend, NetworkInterfaceConfig, NetworkInterfaceUpdateConfig, VirtioConfig};
+pub use virtio_net::VirtioConfig;
+#[cfg(any(
+    feature = "virtio-net",
+    feature = "vhost-net",
+    feature = "vhost-user-net"
+))]
+pub use virtio_net::{Backend, NetworkInterfaceConfig, NetworkInterfaceUpdateConfig};

src/dragonball/src/api/v1/virtio_net.rs

@@ -12,7 +12,13 @@ use serde::{Deserialize, Serialize};
 use super::{VirtioNetDeviceConfigInfo, VirtioNetDeviceConfigUpdateInfo};
 use crate::config_manager::RateLimiterConfigInfo;
 #[cfg(feature = "vhost-net")]
-use crate::device_manager::vhost_net_dev_mgr::{self, VhostNetDeviceConfigInfo};
+use crate::device_manager::vhost_net_dev_mgr;
+#[cfg(feature = "vhost-net")]
+use crate::device_manager::vhost_net_dev_mgr::VhostNetDeviceConfigInfo;
+#[cfg(feature = "vhost-user-net")]
+use crate::device_manager::vhost_user_net_dev_mgr;
+#[cfg(feature = "vhost-user-net")]
+use crate::device_manager::vhost_user_net_dev_mgr::VhostUserNetDeviceConfigInfo;
 #[cfg(feature = "virtio-net")]
 use crate::device_manager::virtio_net_dev_mgr;
 
@@ -28,6 +34,10 @@ pub enum Backend {
     #[cfg(feature = "vhost-net")]
     /// Vhost-net
     Vhost(VirtioConfig),
+    #[serde(rename = "vhost-user")]
+    #[cfg(feature = "vhost-user-net")]
+    /// Vhost-user-net
+    VhostUser(VhostUserConfig),
 }
 
 impl Default for Backend {
@@ -43,6 +53,7 @@ impl Default for Backend {
 }
 
 /// Virtio network config, working for virtio-net and vhost-net.
+#[cfg(any(feature = "virtio-net", feature = "vhost-net"))]
 #[derive(Clone, Debug, Default, Deserialize, PartialEq, Eq, Serialize)]
 pub struct VirtioConfig {
     /// ID of the guest network interface.
@@ -57,6 +68,13 @@ pub struct VirtioConfig {
     pub allow_duplicate_mac: bool,
 }
 
+#[cfg(feature = "vhost-user-net")]
+#[derive(Clone, Debug, Default, Deserialize, PartialEq, Eq, Serialize)]
+pub struct VhostUserConfig {
+    /// Vhost-user socket path.
+    pub sock_path: String,
+}
+
 /// This struct represents the strongly typed equivalent of the json body from
 /// net iface related requests.
 /// This struct works with virtio-net devices and vhost-net devices.
@@ -154,6 +172,39 @@ impl From<&NetworkInterfaceConfig> for VhostNetDeviceConfigInfo {
     }
 }
 
+#[cfg(feature = "vhost-user-net")]
+impl From<NetworkInterfaceConfig> for VhostUserNetDeviceConfigInfo {
+    fn from(value: NetworkInterfaceConfig) -> Self {
+        let self_ref = &value;
+        self_ref.into()
+    }
+}
+#[cfg(feature = "vhost-user-net")]
+impl From<&NetworkInterfaceConfig> for VhostUserNetDeviceConfigInfo {
+    fn from(value: &NetworkInterfaceConfig) -> Self {
+        let num_queues = value
+            .num_queues
+            .unwrap_or(vhost_user_net_dev_mgr::NUM_QUEUES);
+        let queue_size = value
+            .queue_size
+            .unwrap_or(vhost_user_net_dev_mgr::QUEUE_SIZE);
+        // It is safe because we tested the type of config before.
+        #[allow(unreachable_patterns)]
+        let config = match &value.backend {
+            Backend::VhostUser(config) => config,
+            _ => panic!("The virtio backend config is invalid: {:?}", value),
+        };
+        Self {
+            sock_path: config.sock_path.clone(),
+            num_queues,
+            queue_size,
+            guest_mac: value.guest_mac,
+            use_shared_irq: value.use_shared_irq,
+            use_generic_irq: value.use_generic_irq,
+        }
+    }
+}
+
 /// The data fed into a network iface update request. Currently, only the RX and
 /// TX rate limiters can be updated.
 #[derive(Clone, Debug, Deserialize, PartialEq, Eq, Serialize, Default)]
@@ -188,13 +239,13 @@ impl From<&NetworkInterfaceUpdateConfig> for VirtioNetDeviceConfigUpdateInfo {
     }
 }
 
+#[cfg(feature = "virtio-net")]
 #[cfg(test)]
 mod tests {
     use dbs_utils::net::MacAddr;
 
-    use crate::api::v1::Backend;
-
     use super::NetworkInterfaceConfig;
+    use crate::api::v1::Backend;
 
     #[test]
     fn test_network_interface_config() {

src/dragonball/src/api/v1/vmm_action.rs

@@ -40,6 +40,10 @@ pub use crate::device_manager::mem_dev_mgr::{MemDeviceConfigInfo, MemDeviceError
 pub use crate::device_manager::vhost_net_dev_mgr::{
     VhostNetDeviceConfigInfo, VhostNetDeviceError, VhostNetDeviceMgr,
 };
+#[cfg(feature = "vhost-user-net")]
+use crate::device_manager::vhost_user_net_dev_mgr::{
+    VhostUserNetDeviceConfigInfo, VhostUserNetDeviceError, VhostUserNetDeviceMgr,
+};
 #[cfg(feature = "virtio-net")]
 pub use crate::device_manager::virtio_net_dev_mgr::{
     VirtioNetDeviceConfigInfo, VirtioNetDeviceConfigUpdateInfo, VirtioNetDeviceError,
@@ -110,6 +114,11 @@ pub enum VmmActionError {
     /// Vhost-net device relared errors.
     VhostNet(#[source] VhostNetDeviceError),
 
+    #[error("vhost-user-net device error: {0:?}")]
+    #[cfg(feature = "vhost-user-net")]
+    /// Vhost-user-net device relared errors.
+    VhostUserNet(#[source] VhostUserNetDeviceError),
+
     #[cfg(any(feature = "virtio-fs", feature = "vhost-user-fs"))]
     /// The action `InsertFsDevice` failed either because of bad user input or an internal error.
     #[error("virtio-fs device error: {0}")]
@@ -191,7 +200,11 @@ pub enum VmmAction {
     /// are the RX and TX rate limiters.
     UpdateBlockDevice(BlockDeviceConfigUpdateInfo),
 
-    #[cfg(any(feature = "virtio-net", feature = "vhost-net"))]
+    #[cfg(any(
+        feature = "virtio-net",
+        feature = "vhost-net",
+        feature = "vhost-user-net"
+    ))]
     /// Add a new network interface config or update one that already exists using the
     /// `NetworkInterfaceConfig` as input. This action can only be called before the microVM has
     /// booted. The response is sent using the `OutcomeSender`.
@@ -320,12 +333,20 @@ impl VmmService {
             VmmAction::RemoveBlockDevice(drive_id) => {
                 self.remove_block_device(vmm, event_mgr, &drive_id)
             }
-            #[cfg(any(feature = "virtio-net", feature = "vhost-net"))]
+            #[cfg(any(
+                feature = "virtio-net",
+                feature = "vhost-net",
+                feature = "vhost-user-net"
+            ))]
             VmmAction::InsertNetworkDevice(config) => match config.backend {
                 #[cfg(feature = "virtio-net")]
                 Backend::Virtio(_) => self.add_virtio_net_device(vmm, event_mgr, config.into()),
                 #[cfg(feature = "vhost-net")]
                 Backend::Vhost(_) => self.add_vhost_net_device(vmm, event_mgr, config.into()),
+                #[cfg(feature = "vhost-user-net")]
+                Backend::VhostUser(_) => {
+                    self.add_vhost_user_net_device(vmm, event_mgr, config.into())
+                }
             },
             #[cfg(feature = "virtio-net")]
             VmmAction::UpdateNetworkInterface(netif_update) => {
@@ -712,6 +733,30 @@ impl VmmService {
             .map_err(VmmActionError::VhostNet)
     }
 
+    #[cfg(feature = "vhost-user-net")]
+    fn add_vhost_user_net_device(
+        &mut self,
+        vmm: &mut Vmm,
+        event_mgr: &mut EventManager,
+        config: VhostUserNetDeviceConfigInfo,
+    ) -> VmmRequestResult {
+        let vm = vmm.get_vm_mut().ok_or(VmmActionError::InvalidVMID)?;
+        let ctx = vm
+            .create_device_op_context(Some(event_mgr.epoll_manager()))
+            .map_err(|err| {
+                if let StartMicroVmError::MicroVMAlreadyRunning = err {
+                    VmmActionError::VhostUserNet(VhostUserNetDeviceError::UpdateNotAllowedPostBoot)
+                } else if let StartMicroVmError::UpcallServerNotReady = err {
+                    VmmActionError::UpcallServerNotReady
+                } else {
+                    VmmActionError::StartMicroVm(err)
+                }
+            })?;
+        VhostUserNetDeviceMgr::insert_device(vm.device_manager_mut(), ctx, config)
+            .map(|_| VmmData::Empty)
+            .map_err(VmmActionError::VhostUserNet)
+    }
+
     #[cfg(any(feature = "virtio-fs", feature = "vhost-user-fs"))]
     #[instrument(skip(self))]
     fn add_fs_device(&mut self, vmm: &mut Vmm, config: FsDeviceConfigInfo) -> VmmRequestResult {

src/dragonball/src/dbs_virtio_devices/Cargo.toml

@@ -16,6 +16,7 @@ caps = "0.5.3"
 dbs-device = { path = "../dbs_device" }
 dbs-interrupt = { path = "../dbs_interrupt", features = ["kvm-legacy-irq", "kvm-msi-irq"] }
 dbs-utils = { path = "../dbs_utils" }
+dbs-address-space = { path = "../dbs_address_space" }
 epoll = ">=4.3.1, <4.3.2"
 io-uring = "0.5.2"
 fuse-backend-rs = { version = "0.10.5", optional = true }
@@ -54,4 +55,5 @@ virtio-balloon = ["virtio-mmio"]
 vhost = ["virtio-mmio", "vhost-rs/vhost-user-master", "vhost-rs/vhost-kern"]
 vhost-net = ["vhost", "vhost-rs/vhost-net"]
 vhost-user = ["vhost"]
-vhost-user-fs = ["vhost-user"]
+vhost-user-fs = ["vhost-user"]
+vhost-user-net = ["vhost-user"]

src/dragonball/src/dbs_virtio_devices/src/balloon.rs

@@ -755,17 +755,19 @@ pub(crate) mod tests {
     use vmm_sys_util::eventfd::EventFd;
 
     use super::*;
-    use crate::tests::VirtQueue;
+    use crate::tests::{create_address_space, VirtQueue};
 
     fn create_balloon_epoll_handler() -> BalloonEpollHandler<Arc<GuestMemoryMmap>> {
         let mem = Arc::new(GuestMemoryMmap::from_ranges(&[(GuestAddress(0x0), 0x10000)]).unwrap());
         let queues = vec![VirtioQueueConfig::create(128, 0).unwrap()];
         let resources = DeviceResources::new();
         let kvm = Kvm::new().unwrap();
         let vm_fd = Arc::new(kvm.create_vm().unwrap());
+        let address_space = create_address_space();
 
         let config = VirtioDeviceConfig::new(
             mem,
+            address_space,
             vm_fd,
             resources,
             queues,
@@ -877,8 +879,10 @@ pub(crate) mod tests {
             let kvm = Kvm::new().unwrap();
             let vm_fd = Arc::new(kvm.create_vm().unwrap());
             let resources = DeviceResources::new();
+            let address_space = create_address_space();
             let config = VirtioDeviceConfig::<Arc<GuestMemoryMmap<()>>>::new(
                 Arc::new(mem),
+                address_space,
                 vm_fd,
                 resources,
                 queues,
@@ -906,8 +910,10 @@ pub(crate) mod tests {
             let kvm = Kvm::new().unwrap();
             let vm_fd = Arc::new(kvm.create_vm().unwrap());
             let resources = DeviceResources::new();
+            let address_space = create_address_space();
             let config = VirtioDeviceConfig::<Arc<GuestMemoryMmap<()>>>::new(
                 Arc::new(mem),
+                address_space,
                 vm_fd,
                 resources,
                 queues,

src/dragonball/src/dbs_virtio_devices/src/block/device.rs

@@ -381,7 +381,7 @@ mod tests {
     use vmm_sys_util::eventfd::EventFd;
 
     use crate::epoll_helper::*;
-    use crate::tests::{VirtQueue, VIRTQ_DESC_F_NEXT, VIRTQ_DESC_F_WRITE};
+    use crate::tests::{create_address_space, VirtQueue, VIRTQ_DESC_F_NEXT, VIRTQ_DESC_F_WRITE};
     use crate::{Error as VirtioError, VirtioQueueConfig};
 
     use super::*;
@@ -932,8 +932,10 @@ mod tests {
             let kvm = Kvm::new().unwrap();
             let vm_fd = Arc::new(kvm.create_vm().unwrap());
             let resources = DeviceResources::new();
+            let address_space = create_address_space();
             let config = VirtioDeviceConfig::<Arc<GuestMemoryMmap<()>>>::new(
                 Arc::new(mem),
+                address_space,
                 vm_fd,
                 resources,
                 queues,
@@ -968,8 +970,10 @@ mod tests {
             let kvm = Kvm::new().unwrap();
             let vm_fd = Arc::new(kvm.create_vm().unwrap());
             let resources = DeviceResources::new();
+            let address_space = create_address_space();
             let config = VirtioDeviceConfig::<Arc<GuestMemoryMmap<()>>>::new(
                 Arc::new(mem),
+                address_space,
                 vm_fd,
                 resources,
                 queues,
@@ -1003,8 +1007,10 @@ mod tests {
             let kvm = Kvm::new().unwrap();
             let vm_fd = Arc::new(kvm.create_vm().unwrap());
             let resources = DeviceResources::new();
+            let address_space = create_address_space();
             let config = VirtioDeviceConfig::<Arc<GuestMemoryMmap<()>>>::new(
                 Arc::new(mem),
+                address_space,
                 vm_fd,
                 resources,
                 queues,

src/dragonball/src/dbs_virtio_devices/src/device.rs

@@ -19,6 +19,7 @@ use std::io::Write;
 use std::ops::Deref;
 use std::sync::Arc;
 
+use dbs_address_space::AddressSpace;
 use dbs_device::resources::{DeviceResources, ResourceConstraint};
 use dbs_interrupt::{InterruptNotifier, NoopNotifier};
 use dbs_utils::epoll_manager::{EpollManager, EpollSubscriber, SubscriberId};
@@ -175,6 +176,8 @@ pub struct VirtioDeviceConfig<
 > {
     /// `GustMemoryAddress` object to access the guest memory.
     pub vm_as: AS,
+    /// Guest address space
+    pub address_space: AddressSpace,
     /// `VmFd` object for the device to access the hypervisor, such as KVM/HyperV etc.
     pub vm_fd: Arc<VmFd>,
     /// Resources assigned to the Virtio device.
@@ -198,6 +201,7 @@ where
     /// Creates a new `VirtioDeviceConfig` object.
     pub fn new(
         vm_as: AS,
+        address_space: AddressSpace,
         vm_fd: Arc<VmFd>,
         resources: DeviceResources,
         queues: Vec<VirtioQueueConfig<Q>>,
@@ -206,6 +210,7 @@ where
     ) -> Self {
         VirtioDeviceConfig {
             vm_as,
+            address_space,
             vm_fd,
             resources,
             queues,
@@ -576,6 +581,7 @@ pub(crate) mod tests {
     use vm_memory::{GuestMemoryAtomic, GuestMemoryMmap, GuestMemoryRegion, MmapRegion};
 
     use super::*;
+    use crate::tests::create_address_space;
     use crate::{VIRTIO_INTR_CONFIG, VIRTIO_INTR_VRING};
 
     pub fn create_virtio_device_config() -> VirtioDeviceConfig<Arc<GuestMemoryMmap>> {
@@ -606,8 +612,11 @@ pub(crate) mod tests {
             ));
         }
 
+        let address_space = create_address_space();
+
         VirtioDeviceConfig::new(
             mem,
+            address_space,
             vmfd,
             DeviceResources::new(),
             queues,
@@ -873,6 +882,7 @@ pub(crate) mod tests {
         let resources = DeviceResources::new();
         let device_config = VirtioDeviceConfig::new(
             gm,
+            address_space,
             vm_fd,
             resources,
             queues,