runtime-rs: Enhancing DirectVolMount Handling with Patching Support

src/libs/kata-types/src/mount.rs

@@ -483,7 +483,7 @@ impl<H> StorageHandlerManager<H> {
 /// The `volume_path` is base64-url-encoded and then safely joined to the `prefix`
 pub fn join_path(prefix: &str, volume_path: &str) -> Result<PathBuf> {
     if volume_path.is_empty() {
-        return Err(anyhow!("volume path must not be empty"));
+        return Err(anyhow!(std::io::ErrorKind::NotFound));
     }
     let b64_url_encoded_path = base64::encode_config(volume_path.as_bytes(), base64::URL_SAFE);
 

src/runtime-rs/crates/resource/src/volume/block_volume.rs

@@ -4,20 +4,17 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
-use anyhow::{anyhow, Context, Result};
+use anyhow::{Context, Result};
 use async_trait::async_trait;
 use nix::sys::{stat, stat::SFlag};
 use tokio::sync::RwLock;
 
 use super::Volume;
-use crate::volume::utils::{
-    generate_shared_path, get_direct_volume_path, volume_mount_info, DEFAULT_VOLUME_FS_TYPE,
-    KATA_DIRECT_VOLUME_TYPE, KATA_MOUNT_BIND_TYPE,
-};
+use crate::volume::utils::{handle_block_volume, DEFAULT_VOLUME_FS_TYPE, KATA_MOUNT_BIND_TYPE};
 use hypervisor::{
     device::{
         device_manager::{do_handle_device, get_block_driver, DeviceManager},
-        DeviceConfig, DeviceType,
+        DeviceConfig,
     },
     BlockConfig,
 };
@@ -29,7 +26,7 @@ pub(crate) struct BlockVolume {
     device_id: String,
 }
 
-/// BlockVolume for bind-mount block volume and direct block volume
+/// BlockVolume for bind-mount block volume
 impl BlockVolume {
     pub(crate) async fn new(
         d: &RwLock<DeviceManager>,
@@ -38,113 +35,29 @@ impl BlockVolume {
         sid: &str,
     ) -> Result<Self> {
         let mnt_src: &str = &m.source;
-        // default block device fs type: ext4.
-        let mut blk_dev_fstype = DEFAULT_VOLUME_FS_TYPE.to_string();
-
         let block_driver = get_block_driver(d).await;
-
-        let block_device_config = match m.r#type.as_str() {
-            KATA_MOUNT_BIND_TYPE => {
-                let fstat = stat::stat(mnt_src).context(format!("stat {}", m.source))?;
-
-                BlockConfig {
-                    major: stat::major(fstat.st_rdev) as i64,
-                    minor: stat::minor(fstat.st_rdev) as i64,
-                    driver_option: block_driver,
-                    ..Default::default()
-                }
-            }
-            KATA_DIRECT_VOLUME_TYPE => {
-                // get volume mountinfo from mountinfo.json
-                let v = volume_mount_info(mnt_src)
-                    .context("deserde information from mountinfo.json")?;
-                // check volume type
-                if v.volume_type != KATA_DIRECT_VOLUME_TYPE {
-                    return Err(anyhow!("volume type {:?} is invalid", v.volume_type));
-                }
-
-                let fstat = stat::stat(v.device.as_str())
-                    .with_context(|| format!("stat volume device file: {}", v.device.clone()))?;
-                if SFlag::from_bits_truncate(fstat.st_mode) != SFlag::S_IFREG
-                    && SFlag::from_bits_truncate(fstat.st_mode) != SFlag::S_IFBLK
-                {
-                    return Err(anyhow!(
-                        "invalid volume device {:?} for volume type {:?}",
-                        v.device,
-                        v.volume_type
-                    ));
-                }
-
-                blk_dev_fstype = v.fs_type.clone();
-
-                BlockConfig {
-                    path_on_host: v.device,
-                    driver_option: block_driver,
-                    ..Default::default()
-                }
-            }
-            _ => {
-                return Err(anyhow!(
-                    "unsupport direct block volume r#type: {:?}",
-                    m.r#type.as_str()
-                ))
-            }
+        let fstat = stat::stat(mnt_src).context(format!("stat {}", m.source))?;
+        let block_device_config = BlockConfig {
+            major: stat::major(fstat.st_rdev) as i64,
+            minor: stat::minor(fstat.st_rdev) as i64,
+            driver_option: block_driver,
+            ..Default::default()
         };
 
         // create and insert block device into Kata VM
         let device_info = do_handle_device(d, &DeviceConfig::BlockCfg(block_device_config.clone()))
             .await
             .context("do handle device failed.")?;
 
-        // storage
-        let mut storage = agent::Storage {
-            options: if read_only {
-                vec!["ro".to_string()]
-            } else {
-                Vec::new()
-            },
-            ..Default::default()
-        };
-
-        // As the true Block Device wrapped in DeviceType, we need to
-        // get it out from the wrapper, and the device_id will be for
-        // BlockVolume.
-        // safe here, device_info is correct and only unwrap it.
-        let mut device_id = String::new();
-        if let DeviceType::Block(device) = device_info {
-            // blk, mmioblk
-            storage.driver = device.config.driver_option;
-            // /dev/vdX
-            storage.source = device.config.virt_path;
-            device_id = device.device_id;
-        }
-
-        // generate host guest shared path
-        let guest_path = generate_shared_path(m.destination.clone(), read_only, &device_id, sid)
-            .await
-            .context("generate host-guest shared path failed")?;
-        storage.mount_point = guest_path.clone();
-
-        // In some case, dest is device /dev/xxx
-        if m.destination.clone().starts_with("/dev") {
-            storage.fs_type = "bind".to_string();
-            storage.options.append(&mut m.options.clone());
-        } else {
-            // usually, the dest is directory.
-            storage.fs_type = blk_dev_fstype;
-        }
-
-        let mount = oci::Mount {
-            destination: m.destination.clone(),
-            r#type: storage.fs_type.clone(),
-            source: guest_path,
-            options: m.options.clone(),
-        };
+        let block_volume =
+            handle_block_volume(device_info, m, read_only, sid, DEFAULT_VOLUME_FS_TYPE)
+                .await
+                .context("do handle block volume failed")?;
 
         Ok(Self {
-            storage: Some(storage),
-            mount,
-            device_id,
+            storage: Some(block_volume.0),
+            mount: block_volume.1,
+            device_id: block_volume.2,
         })
     }
 }
@@ -178,28 +91,13 @@ impl Volume for BlockVolume {
     }
 }
 
-pub(crate) fn is_block_volume(m: &oci::Mount) -> Result<bool> {
-    let vol_types = [KATA_MOUNT_BIND_TYPE, KATA_DIRECT_VOLUME_TYPE];
-    if !vol_types.contains(&m.r#type.as_str()) {
-        return Ok(false);
+pub(crate) fn is_block_volume(m: &oci::Mount) -> bool {
+    if m.r#type.as_str() != KATA_MOUNT_BIND_TYPE {
+        return false;
     }
 
-    let source = if m.r#type.as_str() == KATA_DIRECT_VOLUME_TYPE {
-        get_direct_volume_path(&m.source).context("get direct volume path failed")?
-    } else {
-        m.source.clone()
-    };
-
-    let fstat =
-        stat::stat(source.as_str()).context(format!("stat mount source {} failed.", source))?;
-    let s_flag = SFlag::from_bits_truncate(fstat.st_mode);
-
-    match m.r#type.as_str() {
-        // case: mount bind and block device
-        KATA_MOUNT_BIND_TYPE if s_flag == SFlag::S_IFBLK => Ok(true),
-        // case: directvol and directory
-        KATA_DIRECT_VOLUME_TYPE if s_flag == SFlag::S_IFDIR => Ok(true),
-        // else: unsupported or todo for other volume type.
-        _ => Ok(false),
+    match stat::stat(m.source.as_str()) {
+        Ok(fstat) => SFlag::from_bits_truncate(fstat.st_mode) == SFlag::S_IFBLK,
+        Err(_) => false,
     }
 }

src/runtime-rs/crates/resource/src/volume/direct_volume.rs

@@ -0,0 +1,121 @@
+// Copyright (c) 2023 Ant Group
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use std::sync::Arc;
+
+use anyhow::{anyhow, Context, Result};
+use kata_types::mount::DirectVolumeMountInfo;
+use nix::sys::{stat, stat::SFlag};
+use tokio::sync::RwLock;
+
+use hypervisor::device::device_manager::DeviceManager;
+
+use crate::volume::{
+    direct_volumes::{
+        get_direct_volume_path, rawblock_volume, spdk_volume, vfio_volume, volume_mount_info,
+        KATA_DIRECT_VOLUME_TYPE, KATA_SPDK_VOLUME_TYPE, KATA_SPOOL_VOLUME_TYPE,
+        KATA_VFIO_VOLUME_TYPE,
+    },
+    utils::KATA_MOUNT_BIND_TYPE,
+    Volume,
+};
+
+enum DirectVolumeType {
+    RawBlock,
+    Spdk,
+    Vfio,
+}
+
+fn to_volume_type(volume_type: &str) -> DirectVolumeType {
+    match volume_type {
+        KATA_SPDK_VOLUME_TYPE | KATA_SPOOL_VOLUME_TYPE => DirectVolumeType::Spdk,
+        KATA_VFIO_VOLUME_TYPE => DirectVolumeType::Vfio,
+        _ => DirectVolumeType::RawBlock,
+    }
+}
+
+pub(crate) async fn handle_direct_volume(
+    d: &RwLock<DeviceManager>,
+    m: &oci::Mount,
+    read_only: bool,
+    sid: &str,
+) -> Result<Option<Arc<dyn Volume>>> {
+    // In the direct volume scenario, we check if the source of a mount is in the
+    // /run/kata-containers/shared/direct-volumes/SID path by iterating over all the mounts.
+    // If the source is not in the path with error kind *NotFound*, we ignore the error
+    // and we treat it as block volume with oci Mount.type *bind*. Just fill in the block
+    // volume info in the DirectVolumeMountInfo
+    let mount_info: DirectVolumeMountInfo = match volume_mount_info(&m.source) {
+        Ok(mount_info) => mount_info,
+        Err(e) => {
+            // First, We need filter the non-io::ErrorKind.
+            if !e.is::<std::io::ErrorKind>() {
+                return Err(anyhow!(format!(
+                    "unexpected error occurs when parse mount info for {:?}, with error {:?}",
+                    &m.source,
+                    e.to_string()
+                )));
+            }
+
+            // Second, we need filter non-NotFound error.
+            // Safe to unwrap here, as the error is of type std::io::ErrorKind.
+            let error_kind = e.downcast_ref::<std::io::ErrorKind>().unwrap();
+            if *error_kind != std::io::ErrorKind::NotFound {
+                return Err(anyhow!(format!(
+                    "failed to parse volume mount info for {:?}, with error {:?}",
+                    &m.source,
+                    e.to_string()
+                )));
+            }
+
+            // Third, if the case is *NotFound* , we just return Ok(None).
+            return Ok(None);
+        }
+    };
+
+    let direct_volume: Arc<dyn Volume> = match to_volume_type(mount_info.volume_type.as_str()) {
+        DirectVolumeType::RawBlock => Arc::new(
+            rawblock_volume::RawblockVolume::new(d, m, &mount_info, read_only, sid)
+                .await
+                .with_context(|| format!("new sid {:?} rawblock volume {:?}", &sid, m))?,
+        ),
+        DirectVolumeType::Spdk => Arc::new(
+            spdk_volume::SPDKVolume::new(d, m, &mount_info, read_only, sid)
+                .await
+                .with_context(|| format!("create spdk volume {:?}", m))?,
+        ),
+        DirectVolumeType::Vfio => Arc::new(
+            vfio_volume::VfioVolume::new(d, m, &mount_info, read_only, sid)
+                .await
+                .with_context(|| format!("new vfio volume {:?}", m))?,
+        ),
+    };
+
+    Ok(Some(direct_volume))
+}
+
+pub(crate) fn is_direct_volume(m: &oci::Mount) -> Result<bool> {
+    let mount_type = m.r#type.as_str();
+    // Filter the non-bind volume and non-direct-vol volume
+    let vol_types = [
+        KATA_MOUNT_BIND_TYPE,
+        KATA_DIRECT_VOLUME_TYPE,
+        KATA_VFIO_VOLUME_TYPE,
+        KATA_SPDK_VOLUME_TYPE,
+        KATA_SPOOL_VOLUME_TYPE,
+    ];
+    if !vol_types.contains(&mount_type) {
+        return Ok(false);
+    }
+
+    match get_direct_volume_path(&m.source) {
+        Ok(directvol_path) => {
+            let fstat = stat::stat(directvol_path.as_str())
+                .context(format!("stat mount source {} failed.", directvol_path))?;
+            Ok(SFlag::from_bits_truncate(fstat.st_mode) == SFlag::S_IFDIR)
+        }
+        Err(_) => Ok(false),
+    }
+}

src/runtime-rs/crates/resource/src/volume/direct_volumes/mod.rs

@@ -0,0 +1,32 @@
+// Copyright (c) 2023 Ant Group
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use anyhow::{Context, Result};
+
+use kata_types::mount::{
+    get_volume_mount_info, join_path, DirectVolumeMountInfo, KATA_DIRECT_VOLUME_ROOT_PATH,
+};
+
+pub mod rawblock_volume;
+pub mod spdk_volume;
+pub mod vfio_volume;
+
+pub const KATA_DIRECT_VOLUME_TYPE: &str = "directvol";
+pub const KATA_VFIO_VOLUME_TYPE: &str = "vfiovol";
+pub const KATA_SPDK_VOLUME_TYPE: &str = "spdkvol";
+pub const KATA_SPOOL_VOLUME_TYPE: &str = "spoolvol";
+
+// volume mount info load infomation from mountinfo.json
+pub fn volume_mount_info(volume_path: &str) -> Result<DirectVolumeMountInfo> {
+    get_volume_mount_info(volume_path)
+}
+
+// get direct volume path whose volume_path encoded with base64
+pub fn get_direct_volume_path(volume_path: &str) -> Result<String> {
+    let volume_full_path =
+        join_path(KATA_DIRECT_VOLUME_ROOT_PATH, volume_path).context("failed to join path.")?;
+
+    Ok(volume_full_path.display().to_string())
+}