runtime-rs: bringing virtio-fs device in device-manager

Currently, virtio-fs is still outside of the device manager.
This causes some management issues, such as the inability to
unify PCI address management.

To do Enhancement of device manager, it will bring virtio-fs
device in device-manager for unified management

Fixes: #7915

Signed-off-by: alex.lyn <alex.lyn@antgroup.com>

src/runtime-rs/crates/hypervisor/src/ch/inner_device.rs

@@ -38,7 +38,11 @@ impl CloudHypervisorInner {
 
     async fn handle_add_device(&mut self, device: DeviceType) -> Result<()> {
         match device {
-            DeviceType::ShareFs(sharefs) => self.handle_share_fs_device(sharefs.config).await,
+            DeviceType::ShareFs(sharefs) => {
+                // It's safe to unwrap as device_config is always there.
+                self.handle_share_fs_device(sharefs.config.device_config.unwrap())
+                    .await
+            }
             DeviceType::HybridVsock(hvsock) => self.handle_hvsock_device(&hvsock.config).await,
             DeviceType::Block(block) => self.handle_block_device(block.config).await,
             _ => Err(anyhow!("unhandled device: {:?}", device)),
@@ -165,7 +169,10 @@ impl CloudHypervisorInner {
         while let Some(dev) = self.pending_devices.pop() {
             match dev {
                 DeviceType::ShareFs(dev) => {
-                    let settings = ShareFsSettings::new(dev.config, self.vm_path.clone());
+                    let settings = ShareFsSettings::new(
+                        dev.config.device_config.unwrap(),
+                        self.vm_path.clone(),
+                    );
 
                     let fs_cfg = FsConfig::try_from(settings)?;
 

src/runtime-rs/crates/hypervisor/src/device/device_manager.rs

@@ -12,8 +12,9 @@ use tokio::sync::{Mutex, RwLock};
 
 use crate::{
     vhost_user_blk::VhostUserBlkDevice, BlockConfig, BlockDevice, HybridVsockDevice, Hypervisor,
-    NetworkDevice, VfioDevice, VhostUserConfig, KATA_BLK_DEV_TYPE, KATA_MMIO_BLK_DEV_TYPE,
-    KATA_NVDIMM_DEV_TYPE, VIRTIO_BLOCK_MMIO, VIRTIO_BLOCK_PCI, VIRTIO_PMEM,
+    NetworkDevice, ShareFsConfig, ShareFsDevice, ShareFsDeviceOp, VfioDevice, VhostUserConfig,
+    KATA_BLK_DEV_TYPE, KATA_MMIO_BLK_DEV_TYPE, KATA_NVDIMM_DEV_TYPE, VIRTIO_BLOCK_MMIO,
+    VIRTIO_BLOCK_PCI, VIRTIO_PMEM,
 };
 
 use super::{
@@ -226,6 +227,11 @@ impl DeviceManager {
                         return Some(device_id.to_string());
                     }
                 }
+                DeviceType::ShareFs(device) => {
+                    if device.config.shared_path == host_path {
+                        return Some(device_id.to_string());
+                    }
+                }
                 _ => {
                     // TODO: support find other device type
                     continue;
@@ -261,7 +267,7 @@ impl DeviceManager {
     async fn new_device(&mut self, device_config: &DeviceConfig) -> Result<String> {
         // device ID must be generated by manager instead of device itself
         // in case of ID collision
-        let device_id = self.new_device_id()?;
+        let mut device_id = self.new_device_id().context("generate device id failed.")?;
         let dev: ArcMutexDevice = match device_config {
             DeviceConfig::BlockCfg(config) => {
                 // try to find the device, if found and just return id.
@@ -324,12 +330,32 @@ impl DeviceManager {
                 // No need to do find device for hybrid vsock device.
                 Arc::new(Mutex::new(HybridVsockDevice::new(&device_id, hvconfig)))
             }
+            DeviceConfig::ShareFsCfg(config) => {
+                // Try to find the sharefs device. If found, just return id with operation Add.
+                if let Some(devid_matched) = self.find_device(config.shared_path.clone()).await {
+                    if config.sharefs_op == ShareFsDeviceOp::Add {
+                        return Ok(devid_matched);
+                    }
+
+                    info!(
+                        sl!(),
+                        "share-fs device with path:{:?} found, device id: {:?}",
+                        config.shared_path.clone(),
+                        devid_matched
+                    );
+                    device_id = devid_matched;
+                }
+
+                self.create_share_fs_device(config, device_id.clone())
+                    .await
+                    .context("failed to create share-fs device")?
+            }
             _ => {
                 return Err(anyhow!("invliad device type"));
             }
         };
 
-        // register device to devices
+        // register device to devices, and it allows update with new device info.
         self.devices.insert(device_id.clone(), dev.clone());
 
         Ok(device_id)
@@ -421,6 +447,27 @@ impl DeviceManager {
         ))))
     }
 
+    async fn create_share_fs_device(
+        &mut self,
+        config: &ShareFsConfig,
+        device_id: String,
+    ) -> Result<ArcMutexDevice> {
+        let mut sharefs_device = ShareFsDevice::new(&device_id, config);
+        if config.sharefs_op == ShareFsDeviceOp::Update {
+            // get the device
+            let sharefs_dev = self
+                .get_device_info(&device_id)
+                .await
+                .context("get sharefs device failed")?;
+            // It's safe to unwrap the sharefs device from the enum DeviceType.
+            if let DeviceType::ShareFs(device) = sharefs_dev {
+                sharefs_device.config.device_config = device.config.device_config;
+            }
+        }
+
+        Ok(Arc::new(Mutex::new(sharefs_device)))
+    }
+
     // device ID must be generated by device manager instead of device itself
     // in case of ID collision
     fn new_device_id(&self) -> Result<String> {

src/runtime-rs/crates/hypervisor/src/device/driver/mod.rs

@@ -20,8 +20,8 @@ pub use virtio_blk::{
     VIRTIO_BLOCK_MMIO, VIRTIO_BLOCK_PCI, VIRTIO_PMEM,
 };
 pub use virtio_fs::{
-    ShareFsDevice, ShareFsDeviceConfig, ShareFsMountConfig, ShareFsMountDevice, ShareFsMountType,
-    ShareFsOperation,
+    ShareFsConfig, ShareFsDevice, ShareFsDeviceConfig, ShareFsDeviceOp, ShareFsMountConfig,
+    ShareFsMountDevice, ShareFsMountType, ShareFsOperation,
 };
 pub use virtio_net::{Address, NetworkConfig, NetworkDevice};
 pub use virtio_vsock::{

src/runtime-rs/crates/hypervisor/src/device/driver/virtio_fs.rs

@@ -4,6 +4,11 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
+use anyhow::{Context, Result};
+use async_trait::async_trait;
+
+use crate::device::{hypervisor, Device, DeviceType};
+
 #[derive(Copy, Clone, Debug)]
 pub enum ShareFsOperation {
     Mount,
@@ -72,7 +77,85 @@ pub struct ShareFsDeviceConfig {
     pub options: Vec<String>,
 }
 
+#[derive(Clone, Debug)]
+pub struct ShareFsConfig {
+    /// shared_path: the upperdir of the passthrough fs exported dir or rafs meta file of rafs
+    pub shared_path: String,
+
+    /// sharefs device option type: Add and Update.
+    pub sharefs_op: ShareFsDeviceOp,
+
+    /// device config for sharefs device
+    pub device_config: Option<ShareFsDeviceConfig>,
+
+    /// mount config for sharefs mount/umount/update
+    pub mount_config: Option<ShareFsMountConfig>,
+}
+
 #[derive(Debug, Clone)]
 pub struct ShareFsDevice {
-    pub config: ShareFsDeviceConfig,
+    /// device id for sharefs device in device manager
+    pub device_id: String,
+
+    /// config for sharefs device
+    pub config: ShareFsConfig,
+}
+
+#[derive(Clone, Debug, PartialEq)]
+pub enum ShareFsDeviceOp {
+    /// Add for sharefs device with add_device()
+    Add,
+
+    /// Update for sharefs mount with update_device()
+    Update,
+}
+
+impl ShareFsDevice {
+    // new creates a share-fs device
+    pub fn new(device_id: &str, config: &ShareFsConfig) -> Self {
+        Self {
+            device_id: device_id.to_string(),
+            config: config.clone(),
+        }
+    }
+}
+
+#[async_trait]
+impl Device for ShareFsDevice {
+    async fn attach(&mut self, h: &dyn hypervisor) -> Result<()> {
+        match self.config.sharefs_op {
+            ShareFsDeviceOp::Add => h
+                .add_device(DeviceType::ShareFs(self.clone()))
+                .await
+                .context("add share-fs device.")?,
+            ShareFsDeviceOp::Update => h
+                .update_device(DeviceType::ShareFs(self.clone()))
+                .await
+                .context("update share-fs device.")?,
+        };
+
+        return Ok(());
+    }
+
+    async fn detach(&mut self, _h: &dyn hypervisor) -> Result<Option<u64>> {
+        // no need to detach share-fs device
+
+        Ok(None)
+    }
+
+    async fn get_device_info(&self) -> DeviceType {
+        DeviceType::ShareFs(self.clone())
+    }
+
+    async fn increase_attach_count(&mut self) -> Result<bool> {
+        // share-fs devices will not be attached multiple times, Just return Ok(false)
+
+        Ok(false)
+    }
+
+    async fn decrease_attach_count(&mut self) -> Result<bool> {
+        // share-fs devices will not be detached multiple times, Just return Ok(false)
+
+        Ok(false)
+    }
 }

src/runtime-rs/crates/hypervisor/src/device/mod.rs

@@ -9,8 +9,8 @@ use std::fmt;
 use crate::device::driver::vhost_user_blk::VhostUserBlkDevice;
 use crate::{
     BlockConfig, BlockDevice, HybridVsockConfig, HybridVsockDevice, Hypervisor as hypervisor,
-    NetworkConfig, NetworkDevice, ShareFsDevice, ShareFsDeviceConfig, ShareFsMountConfig,
-    ShareFsMountDevice, VfioConfig, VfioDevice, VhostUserConfig, VsockConfig, VsockDevice,
+    NetworkConfig, NetworkDevice, ShareFsConfig, ShareFsDevice, VfioConfig, VfioDevice,
+    VhostUserConfig, VsockConfig, VsockDevice,
 };
 use anyhow::Result;
 use async_trait::async_trait;
@@ -24,9 +24,9 @@ pub enum DeviceConfig {
     BlockCfg(BlockConfig),
     VhostUserBlkCfg(VhostUserConfig),
     NetworkCfg(NetworkConfig),
-    ShareFsCfg(ShareFsDeviceConfig),
+    ShareFsCfg(ShareFsConfig),
     VfioCfg(VfioConfig),
-    ShareFsMountCfg(ShareFsMountConfig),
+    // ShareFsMountCfg(ShareFsMountConfig),
     VsockCfg(VsockConfig),
     HybridVsockCfg(HybridVsockConfig),
 }
@@ -38,7 +38,7 @@ pub enum DeviceType {
     Vfio(VfioDevice),
     Network(NetworkDevice),
     ShareFs(ShareFsDevice),
-    ShareFsMount(ShareFsMountDevice),
+    // ShareFsMount(ShareFsMountDevice),
     HybridVsock(HybridVsockDevice),
     Vsock(VsockDevice),
 }

src/runtime-rs/crates/hypervisor/src/dragonball/inner_device.rs

@@ -68,12 +68,11 @@ impl DragonballInner {
                 )
                 .context("add vhost user based block device"),
             DeviceType::HybridVsock(hvsock) => self.add_hvsock(&hvsock.config).context("add vsock"),
-            DeviceType::ShareFs(sharefs) => self
-                .add_share_fs_device(&sharefs.config)
-                .context("add share fs device"),
-            DeviceType::ShareFsMount(sharefs_mount) => self
-                .add_share_fs_mount(&sharefs_mount.config)
-                .context("add share fs mount"),
+            DeviceType::ShareFs(sharefs) => {
+                // It's safe to unwrap device config as device_config is always there.
+                self.add_share_fs_device(&sharefs.config.device_config.unwrap())
+                    .context("add share fs device")
+            }
             DeviceType::Vsock(_) => {
                 todo!()
             }
@@ -110,8 +109,14 @@ impl DragonballInner {
 
     pub(crate) async fn update_device(&mut self, device: DeviceType) -> Result<()> {
         info!(sl!(), "dragonball update device {:?}", &device);
-
-        Ok(())
+        match device {
+            DeviceType::ShareFs(sharefs_mount) => {
+                // It's safe to unwrap device config as device_config is always there.
+                self.add_share_fs_mount(&sharefs_mount.config.mount_config.unwrap())
+                    .context("update share-fs device with mount operation.")
+            }
+            _ => Err(anyhow!("unsupported device {:?} to update.", device)),
+        }
     }
 
     fn add_vfio_device(&mut self, device: &VfioDevice) -> Result<()> {

src/runtime-rs/crates/hypervisor/src/qemu/inner.rs

@@ -159,6 +159,6 @@ impl QemuInner {
     pub(crate) async fn update_device(&mut self, device: DeviceType) -> Result<()> {
         info!(sl!(), "QemuInner::update_device() {:?}", &device);
 
-        Ok(())
+        todo!()
     }
 }

src/runtime-rs/crates/resource/src/manager_inner.rs

@@ -102,7 +102,10 @@ impl ResourceManagerInner {
                     {
                         let share_fs = share_fs::new(&self.sid, &c).context("new share fs")?;
                         share_fs
-                            .setup_device_before_start_vm(self.hypervisor.as_ref())
+                            .setup_device_before_start_vm(
+                                self.hypervisor.as_ref(),
+                                &self.device_manager,
+                            )
                             .await
                             .context("setup share fs device before start vm")?;
 
@@ -212,7 +215,7 @@ impl ResourceManagerInner {
     pub async fn setup_after_start_vm(&mut self) -> Result<()> {
         if let Some(share_fs) = self.share_fs.as_ref() {
             share_fs
-                .setup_device_after_start_vm(self.hypervisor.as_ref())
+                .setup_device_after_start_vm(self.hypervisor.as_ref(), &self.device_manager)
                 .await
                 .context("setup share fs device after start vm")?;
         }
@@ -227,6 +230,7 @@ impl ResourceManagerInner {
                 .context("handle neighbors")?;
             self.handle_routes(network).await.context("handle routes")?;
         }
+
         Ok(())
     }
 

src/runtime-rs/crates/resource/src/rootfs/mod.rs

@@ -22,6 +22,7 @@ use self::{block_rootfs::is_block_rootfs, nydus_rootfs::NYDUS_ROOTFS_TYPE};
 const ROOTFS: &str = "rootfs";
 const HYBRID_ROOTFS_LOWER_DIR: &str = "rootfs_lower";
 const TYPE_OVERLAY_FS: &str = "overlay";
+
 #[async_trait]
 pub trait Rootfs: Send + Sync {
     async fn get_guest_rootfs_path(&self) -> Result<String>;
@@ -101,9 +102,16 @@ impl RootFsResource {
                     // handle nydus rootfs
                     let share_rootfs: Arc<dyn Rootfs> = if layer.fs_type == NYDUS_ROOTFS_TYPE {
                         Arc::new(
-                            nydus_rootfs::NydusRootfs::new(share_fs, h, sid, cid, layer)
-                                .await
-                                .context("new nydus rootfs")?,
+                            nydus_rootfs::NydusRootfs::new(
+                                device_manager,
+                                share_fs,
+                                h,
+                                sid,
+                                cid,
+                                layer,
+                            )
+                            .await
+                            .context("new nydus rootfs")?,
                         )
                     }
                     // handle sharefs rootfs

src/runtime-rs/crates/resource/src/rootfs/nydus_rootfs.rs

@@ -39,6 +39,7 @@ pub(crate) struct NydusRootfs {
 
 impl NydusRootfs {
     pub async fn new(
+        d: &RwLock<DeviceManager>,
         share_fs: &Arc<dyn ShareFs>,
         h: &dyn Hypervisor,
         sid: &str,
@@ -61,7 +62,8 @@ impl NydusRootfs {
                 // rafs mount the metadata of nydus rootfs
                 let rafs_mnt = do_get_guest_share_path(HYBRID_ROOTFS_LOWER_DIR, cid, true);
                 rafs_mount(
-                    h,
+                    d,
+                    sid,
                     rafs_meta.to_string(),
                     rafs_mnt,
                     extra_options.config.clone(),