runtime: agent: use up to 10 PCI segments (#61)

Attach block devices to segments 1-9, and use segment 0 for other
types of devices.

Signed-off-by: Dan Mihai <dmihai@microsoft.com>

src/agent/src/device.rs

@@ -147,7 +147,9 @@ where
 // provided.
 #[instrument]
 pub fn pcipath_to_sysfs(root_bus_sysfs: &str, pcipath: &pci::Path) -> Result<String> {
-    let mut bus = "0000:00".to_string();
+    // Support up to 10 PCI segments.
+    let mut bus = "000[0-9]:00".to_string();
+
     let mut relpath = String::new();
 
     for i in 0..pcipath.len() {
@@ -225,7 +227,7 @@ struct VirtioBlkPciMatcher {
 
 impl VirtioBlkPciMatcher {
     fn new(relpath: &str) -> VirtioBlkPciMatcher {
-        let root_bus = create_pci_root_bus_path();
+        let root_bus = create_pci_root_bus_pattern();
         let re = format!(r"^{}{}/virtio[0-9]+/block/", root_bus, relpath);
 
         VirtioBlkPciMatcher {
@@ -245,7 +247,7 @@ pub async fn get_virtio_blk_pci_device_name(
     sandbox: &Arc<Mutex<Sandbox>>,
     pcipath: &pci::Path,
 ) -> Result<String> {
-    let root_bus_sysfs = format!("{}{}", SYSFS_DIR, create_pci_root_bus_path());
+    let root_bus_sysfs = format!("{}{}", SYSFS_DIR, create_pci_root_bus_pattern());
     let sysfs_rel_path = pcipath_to_sysfs(&root_bus_sysfs, pcipath)?;
     let matcher = VirtioBlkPciMatcher::new(&sysfs_rel_path);
 
@@ -347,7 +349,7 @@ struct PciMatcher {
 
 impl PciMatcher {
     fn new(relpath: &str) -> Result<PciMatcher> {
-        let root_bus = create_pci_root_bus_path();
+        let root_bus = create_pci_root_bus_pattern();
         Ok(PciMatcher {
             devpath: format!("{}{}", root_bus, relpath),
         })
@@ -364,7 +366,7 @@ pub async fn wait_for_pci_device(
     sandbox: &Arc<Mutex<Sandbox>>,
     pcipath: &pci::Path,
 ) -> Result<pci::Address> {
-    let root_bus_sysfs = format!("{}{}", SYSFS_DIR, create_pci_root_bus_path());
+    let root_bus_sysfs = format!("{}{}", SYSFS_DIR, create_pci_root_bus_pattern());
     let sysfs_rel_path = pcipath_to_sysfs(&root_bus_sysfs, pcipath)?;
     let matcher = PciMatcher::new(&sysfs_rel_path)?;
 
@@ -1487,7 +1489,7 @@ mod tests {
     #[tokio::test]
     async fn test_get_device_name() {
         let devname = "vda";
-        let root_bus = create_pci_root_bus_path();
+        let root_bus = create_pci_root_bus_pattern();
         let relpath = "/0000:00:0a.0/0000:03:0b.0";
         let devpath = format!("{}{}/virtio4/block/{}", root_bus, relpath, devname);
 
@@ -1522,7 +1524,7 @@ mod tests {
     #[tokio::test]
     #[allow(clippy::redundant_clone)]
     async fn test_virtio_blk_matcher() {
-        let root_bus = create_pci_root_bus_path();
+        let root_bus = create_pci_root_bus_pattern();
         let devname = "vda";
 
         let mut uev_a = crate::uevent::Uevent::default();
@@ -1607,7 +1609,7 @@ mod tests {
     #[tokio::test]
     #[allow(clippy::redundant_clone)]
     async fn test_scsi_block_matcher() {
-        let root_bus = create_pci_root_bus_path();
+        let root_bus = create_pci_root_bus_pattern();
         let devname = "sda";
 
         let mut uev_a = crate::uevent::Uevent::default();

src/agent/src/linux_abi.rs

@@ -17,12 +17,13 @@ pub const SYSFS_DIR: &str = "/sys";
     target_arch = "x86_64",
     target_arch = "x86"
 ))]
-pub fn create_pci_root_bus_path() -> String {
-    String::from("/devices/pci0000:00")
+pub fn create_pci_root_bus_pattern() -> String {
+    // Support up to 10 PCI segments.
+    String::from("/devices/pci000[0-9]:00")
 }
 
 #[cfg(target_arch = "aarch64")]
-pub fn create_pci_root_bus_path() -> String {
+pub fn create_pci_root_bus_pattern() -> String {
     let ret = String::from("/devices/platform/4010000000.pcie/pci0000:00");
 
     let acpi_root_bus_path = String::from("/devices/pci0000:00");

src/runtime/virtcontainers/clh.go

@@ -72,8 +72,8 @@ const (
 const (
 	// Values are mandatory by http API
 	// Values based on:
-	clhTimeout                     = 10
-	clhAPITimeout                  = 1
+	clhTimeout    = 10
+	clhAPITimeout = 1
 
 	// TODO: reduce the SEV-SNP Guest boot time.
 	//
@@ -878,7 +878,17 @@ func clhDriveIndexToID(i int) string {
 // assumption convert a clh PciDeviceInfo into a PCI path
 func clhPciInfoToPath(pciInfo chclient.PciDeviceInfo) (types.PciPath, error) {
 	tokens := strings.Split(pciInfo.Bdf, ":")
-	if len(tokens) != 3 || tokens[0] != "0000" || tokens[1] != "00" {
+	if len(tokens) != 3 || tokens[1] != "00" {
+		return types.PciPath{}, fmt.Errorf("Unexpected PCI address %q from clh hotplug", pciInfo.Bdf)
+	}
+
+	// Support up to 10 PCI segments.
+	pciSegment, err := regexp.Compile(`^000[0-9]$`)
+	if err != nil {
+		return types.PciPath{}, fmt.Errorf("Internal error: cannot compile PCI segment regex")
+	}
+
+	if !pciSegment.MatchString(tokens[0]) {
 		return types.PciPath{}, fmt.Errorf("Unexpected PCI address %q from clh hotplug", pciInfo.Bdf)
 	}
 
@@ -931,6 +941,10 @@ func (clh *cloudHypervisor) hotplugAddBlockDevice(drive *config.BlockDrive) erro
 		clhDisk.SetRateLimiterConfig(*diskRateLimiterConfig)
 	}
 
+	// Hotplug block devices on PCI segments >= 1. PCI segment 0 is used
+	// for the network interface, any disks present at Guest boot time, etc.
+	clhDisk.SetPciSegment(int32(drive.Index)/32 + 1)
+
 	pciInfo, _, err := cl.VmAddDiskPut(ctx, clhDisk)
 
 	if err != nil {