diff --git a/pci/src/bus.rs b/pci/src/bus.rs index fd19321de5..d96891beec 100644 --- a/pci/src/bus.rs +++ b/pci/src/bus.rs @@ -45,7 +45,7 @@ pub enum PciRootError { #[error("Invalid PCI device identifier provided")] InvalidPciDeviceSlot(usize), /// Valid PCI device identifier but already used. - #[error("Valid PCI device identifier but already used")] + #[error("Valid PCI device identifier but already used: {0}")] AlreadyInUsePciDeviceSlot(usize), } pub type Result = std::result::Result; @@ -166,15 +166,42 @@ impl PciBus { Ok(()) } - pub fn next_device_id(&mut self) -> Result { - for (idx, device_id) in self.device_ids.iter_mut().enumerate() { - if !(*device_id) { - *device_id = true; - return Ok(idx as u32); + /// Allocates a PCI device ID on the bus. + /// + /// - `id`: ID to allocate on the bus. If [`None`], the next free + /// device ID on the bus is allocated, else the ID given is + /// allocated + /// + /// ## Errors + /// * Returns [`PciRootError::AlreadyInUsePciDeviceSlot`] in case + /// the ID requested is already allocated. + /// * Returns [`PciRootError::InvalidPciDeviceSlot`] in case the + /// requested ID exceeds the maximum number of devices allowed per + /// bus (see [`NUM_DEVICE_IDS`]). + /// * If `id` is [`None`]: Returns + /// [`PciRootError::NoPciDeviceSlotAvailable`] if no free device + /// slot is available on the bus. + pub fn allocate_device_id(&mut self, id: Option) -> Result { + if let Some(id) = id { + if (id as usize) < NUM_DEVICE_IDS { + if !self.device_ids[id as usize] { + self.device_ids[id as usize] = true; + Ok(id as u32) + } else { + Err(PciRootError::AlreadyInUsePciDeviceSlot(id as usize)) + } + } else { + Err(PciRootError::InvalidPciDeviceSlot(id as usize)) } + } else { + for (idx, device_id) in self.device_ids.iter_mut().enumerate() { + if !(*device_id) { + *device_id = true; + return Ok(idx as u32); + } + } + Err(PciRootError::NoPciDeviceSlotAvailable) } - - Err(PciRootError::NoPciDeviceSlotAvailable) } pub fn get_device_id(&mut self, id: usize) -> Result<()> { @@ -484,3 +511,94 @@ fn parse_io_config_address(config_address: u32) -> (usize, usize, usize, usize) shift_and_mask(config_address, REGISTER_NUMBER_OFFSET, REGISTER_NUMBER_MASK), ) } + +#[cfg(test)] +mod tests { + use std::error::Error; + use std::result::Result; + + use super::*; + + #[derive(Debug)] + struct MocRelocDevice; + + impl DeviceRelocation for MocRelocDevice { + fn move_bar( + &self, + _old_base: u64, + _new_base: u64, + _len: u64, + _pci_dev: &mut dyn PciDevice, + _region_type: PciBarRegionType, + ) -> Result<(), std::io::Error> { + Ok(()) + } + } + + fn setup_bus() -> PciBus { + let pci_root = PciRoot::new(None); + let moc_device_reloc = Arc::new(MocRelocDevice {}); + PciBus::new(pci_root, moc_device_reloc) + } + + #[test] + // Test to acquire all IDs that can be acquired + fn allocate_device_id_next_free() { + // The first address is occupied by the root + let mut bus = setup_bus(); + for expected_id in 1..NUM_DEVICE_IDS { + assert_eq!(expected_id as u32, bus.allocate_device_id(None).unwrap()); + } + } + + #[test] + // Test that requesting specific ID work + fn allocate_device_id_request_id() -> Result<(), Box> { + // The first address is occupied by the root + let mut bus = setup_bus(); + let max_id = (NUM_DEVICE_IDS - 1).try_into()?; + assert_eq!(0x01_u32, bus.allocate_device_id(Some(0x01))?); + assert_eq!(0x10_u32, bus.allocate_device_id(Some(0x10))?); + assert_eq!(max_id as u32, bus.allocate_device_id(Some(max_id))?); + Ok(()) + } + + #[test] + // Test that requesting the same ID twice fails + fn allocate_device_id_request_id_twice_fails() -> Result<(), Box> { + let mut bus = setup_bus(); + let max_id = (NUM_DEVICE_IDS - 1).try_into()?; + bus.allocate_device_id(Some(max_id))?; + let _result = bus.allocate_device_id(Some(max_id)); + assert!(matches!( + PciRootError::AlreadyInUsePciDeviceSlot(max_id.into()), + _result + )); + Ok(()) + } + + #[test] + // Test to request an invalid ID + fn allocate_device_id_request_invalid_id_fails() -> Result<(), Box> { + let mut bus = setup_bus(); + let max_id = (NUM_DEVICE_IDS + 1).try_into()?; + let _result = bus.allocate_device_id(Some(max_id)); + assert!(matches!( + PciRootError::InvalidPciDeviceSlot(max_id.into()), + _result + )); + Ok(()) + } + + #[test] + // Test to acquire an ID when all IDs were already acquired + fn allocate_device_id_none_left() { + // The first address is occupied by the root + let mut bus = setup_bus(); + for expected_id in 1..NUM_DEVICE_IDS { + assert_eq!(expected_id as u32, bus.allocate_device_id(None).unwrap()); + } + let _result = bus.allocate_device_id(None); + assert!(matches!(PciRootError::NoPciDeviceSlotAvailable, _result)); + } +} diff --git a/src/main.rs b/src/main.rs index d096f28cb1..e05eff9abc 100644 --- a/src/main.rs +++ b/src/main.rs @@ -993,6 +993,7 @@ mod unit_tests { rng: RngConfig { src: PathBuf::from("/dev/urandom"), iommu: false, + bdf_device_id: None, }, balloon: None, fs: None, @@ -1003,6 +1004,7 @@ mod unit_tests { iommu: false, socket: None, url: None, + bdf_device_id: None, }, console: ConsoleConfig { file: None, @@ -1010,6 +1012,7 @@ mod unit_tests { iommu: false, socket: None, url: None, + bdf_device_id: None, }, #[cfg(target_arch = "x86_64")] debug_console: DebugConsoleConfig::default(), @@ -1211,6 +1214,24 @@ mod unit_tests { }"#, true, ), + ( + vec![ + "cloud-hypervisor", + "--kernel", + "/path/to/kernel", + "--disk", + "path=/path/to/disk/1,addr=15.0", + "path=/path/to/disk/2", + ], + r#"{ + "payload": {"kernel": "/path/to/kernel"}, + "disks": [ + {"path": "/path/to/disk/1", "bdf_device_id": 15}, + {"path": "/path/to/disk/2"} + ] + }"#, + true, + ), ( vec![ "cloud-hypervisor", @@ -1422,6 +1443,20 @@ mod unit_tests { }"#, true, ), + ( + vec![ + "cloud-hypervisor", "--kernel", "/path/to/kernel", + "--net", + "mac=12:34:56:78:90:ab,host_mac=34:56:78:90:ab:cd,tap=tap0,ip=1.2.3.4,mask=5.6.7.8,addr=15.0", + ], + r#"{ + "payload": {"kernel": "/path/to/kernel"}, + "net": [ + {"mac": "12:34:56:78:90:ab", "host_mac": "34:56:78:90:ab:cd", "tap": "tap0", "ip": "1.2.3.4", "mask": "5.6.7.8", "num_queues": 2, "queue_size": 256, "bdf_device_id": 15} + ] + }"#, + true, + ), #[cfg(target_arch = "x86_64")] ( vec![ @@ -1493,11 +1528,11 @@ mod unit_tests { "--kernel", "/path/to/kernel", "--rng", - "src=/path/to/entropy/source", + "src=/path/to/entropy/source,addr=15.0", ], r#"{ "payload": {"kernel": "/path/to/kernel"}, - "rng": {"src": "/path/to/entropy/source"} + "rng": {"src": "/path/to/entropy/source", "bdf_device_id": 15} }"#, true, )] @@ -1514,14 +1549,14 @@ mod unit_tests { "cloud-hypervisor", "--kernel", "/path/to/kernel", "--memory", "shared=true", "--fs", - "tag=virtiofs1,socket=/path/to/sock1", + "tag=virtiofs1,socket=/path/to/sock1,addr=15.0", "tag=virtiofs2,socket=/path/to/sock2", ], r#"{ "payload": {"kernel": "/path/to/kernel"}, "memory" : { "shared": true, "size": 536870912 }, "fs": [ - {"tag": "virtiofs1", "socket": "/path/to/sock1"}, + {"tag": "virtiofs1", "socket": "/path/to/sock1", "bdf_device_id": 15}, {"tag": "virtiofs2", "socket": "/path/to/sock2"} ] }"#, @@ -1593,13 +1628,13 @@ mod unit_tests { "--kernel", "/path/to/kernel", "--pmem", - "file=/path/to/img/1,size=1G", + "file=/path/to/img/1,size=1G,addr=15.0", "file=/path/to/img/2,size=2G", ], r#"{ "payload": {"kernel": "/path/to/kernel"}, "pmem": [ - {"file": "/path/to/img/1", "size": 1073741824}, + {"file": "/path/to/img/1", "size": 1073741824,"bdf_device_id": 15}, {"file": "/path/to/img/2", "size": 2147483648} ] }"#, @@ -1877,13 +1912,13 @@ mod unit_tests { "--kernel", "/path/to/kernel", "--vdpa", - "path=/path/to/device/1", + "path=/path/to/device/1,addr=15.0", "path=/path/to/device/2,num_queues=2", ], r#"{ "payload": {"kernel": "/path/to/kernel"}, "vdpa": [ - {"path": "/path/to/device/1", "num_queues": 1}, + {"path": "/path/to/device/1", "num_queues": 1, "bdf_device_id": 15}, {"path": "/path/to/device/2", "num_queues": 2} ] }"#, @@ -1922,11 +1957,11 @@ mod unit_tests { "--kernel", "/path/to/kernel", "--vsock", - "cid=123,socket=/path/to/sock/1", + "cid=123,socket=/path/to/sock/1,addr=15.0", ], r#"{ "payload": {"kernel": "/path/to/kernel"}, - "vsock": {"cid": 123, "socket": "/path/to/sock/1"} + "vsock": {"cid": 123, "socket": "/path/to/sock/1", "bdf_device_id": 15} }"#, true, ), diff --git a/vmm/src/config.rs b/vmm/src/config.rs index 00d26ec881..2a9de2f567 100644 --- a/vmm/src/config.rs +++ b/vmm/src/config.rs @@ -1068,6 +1068,26 @@ impl RateLimiterGroupConfig { } } +fn parse_addr( + s: &str, +) -> std::result::Result<(u8 /* device */, u8 /* function */), OptionParserError> { + let (device_str, function_str) = s + .split_once('.') + .ok_or(OptionParserError::InvalidSyntax(s.to_owned()))?; + let device_id = device_str + .parse::() + .map_err(|_| OptionParserError::InvalidSyntax(s.to_owned()))?; + let function_id = function_str + .parse::() + .map_err(|_| OptionParserError::InvalidSyntax(s.to_owned()))?; + if function_id != 0 { + return Err(OptionParserError::InvalidValue(format!( + "invalid device function ID: {function_id}" + ))); + } + Ok((device_id, function_id)) +} + impl DiskConfig { pub const SYNTAX: &'static str = "Disk parameters \ \"path=,readonly=on|off,direct=on|off,iommu=on|off,\ @@ -1077,7 +1097,7 @@ impl DiskConfig { ops_size=,ops_one_time_burst=,ops_refill_time=,\ id=,pci_segment=,rate_limit_group=,\ queue_affinity=,\ - serial="; + serial=,addr="; pub fn parse(disk: &str) -> Result { let mut parser = OptionParser::new(); @@ -1102,7 +1122,8 @@ impl DiskConfig { .add("pci_segment") .add("serial") .add("rate_limit_group") - .add("queue_affinity"); + .add("queue_affinity") + .add("addr"); parser.parse(disk).map_err(Error::ParseDisk)?; let path = parser.get("path").map(PathBuf::from); @@ -1214,6 +1235,13 @@ impl DiskConfig { None }; + let (bdf_device_id, _bdf_function) = if let Some(s) = parser.get("addr") { + let (a, b) = parse_addr(s.as_str()).map_err(Error::ParseDisk)?; + (Some(a), Some(b)) + } else { + (None, None) + }; + Ok(DiskConfig { path, readonly, @@ -1231,6 +1259,7 @@ impl DiskConfig { pci_segment, serial, queue_affinity, + bdf_device_id, }) } @@ -1302,7 +1331,7 @@ impl NetConfig { vhost_user=,socket=,vhost_mode=client|server,\ bw_size=,bw_one_time_burst=,bw_refill_time=,\ ops_size=,ops_one_time_burst=,ops_refill_time=,pci_segment=\ - offload_tso=on|off,offload_ufo=on|off,offload_csum=on|off\""; + offload_tso=on|off,offload_ufo=on|off,offload_csum=on|off,addr=DD.F\""; pub fn parse(net: &str) -> Result { let mut parser = OptionParser::new(); @@ -1331,7 +1360,8 @@ impl NetConfig { .add("ops_size") .add("ops_one_time_burst") .add("ops_refill_time") - .add("pci_segment"); + .add("pci_segment") + .add("addr"); parser.parse(net).map_err(Error::ParseNetwork)?; let tap = parser.get("tap"); @@ -1447,6 +1477,13 @@ impl NetConfig { None }; + let (bdf_device_id, _bdf_function) = if let Some(s) = parser.get("addr") { + let (a, b) = parse_addr(s.as_str()).map_err(Error::ParseDisk)?; + (Some(a), Some(b)) + } else { + (None, None) + }; + let config = NetConfig { tap, ip, @@ -1467,6 +1504,7 @@ impl NetConfig { offload_tso, offload_ufo, offload_csum, + bdf_device_id, }; Ok(config) } @@ -1531,7 +1569,7 @@ impl NetConfig { impl RngConfig { pub fn parse(rng: &str) -> Result { let mut parser = OptionParser::new(); - parser.add("src").add("iommu"); + parser.add("src").add("iommu").add("addr"); parser.parse(rng).map_err(Error::ParseRng)?; let src = PathBuf::from( @@ -1545,19 +1583,30 @@ impl RngConfig { .unwrap_or(Toggle(false)) .0; - Ok(RngConfig { src, iommu }) + let (bdf_device_id, _bdf_function) = if let Some(s) = parser.get("addr") { + let (a, b) = parse_addr(s.as_str()).map_err(Error::ParseDisk)?; + (Some(a), Some(b)) + } else { + (None, None) + }; + + Ok(RngConfig { + src, + iommu, + bdf_device_id, + }) } } impl BalloonConfig { pub const SYNTAX: &'static str = "Balloon parameters \"size=,deflate_on_oom=on|off,\ - free_page_reporting=on|off\""; + free_page_reporting=on|off,addr=\""; pub fn parse(balloon: &str) -> Result { let mut parser = OptionParser::new(); parser.add("size"); parser.add("deflate_on_oom"); - parser.add("free_page_reporting"); + parser.add("free_page_reporting").add("addr"); parser.parse(balloon).map_err(Error::ParseBalloon)?; let size = parser @@ -1578,10 +1627,18 @@ impl BalloonConfig { .unwrap_or(Toggle(false)) .0; + let (bdf_device_id, _bdf_function) = if let Some(s) = parser.get("addr") { + let (a, b) = parse_addr(s.as_str()).map_err(Error::ParseDisk)?; + (Some(a), Some(b)) + } else { + (None, None) + }; + Ok(BalloonConfig { size, deflate_on_oom, free_page_reporting, + bdf_device_id, }) } } @@ -1589,7 +1646,8 @@ impl BalloonConfig { impl FsConfig { pub const SYNTAX: &'static str = "virtio-fs parameters \ \"tag=,socket=,num_queues=,\ - queue_size=,id=,pci_segment=\""; + queue_size=,id=,pci_segment=,\ + addr=\""; pub fn parse(fs: &str) -> Result { let mut parser = OptionParser::new(); @@ -1599,7 +1657,8 @@ impl FsConfig { .add("num_queues") .add("socket") .add("id") - .add("pci_segment"); + .add("pci_segment") + .add("addr"); parser.parse(fs).map_err(Error::ParseFileSystem)?; let tag = parser.get("tag").ok_or(Error::ParseFsTagMissing)?; @@ -1624,6 +1683,13 @@ impl FsConfig { .map_err(Error::ParseFileSystem)? .unwrap_or_default(); + let (bdf_device_id, _bdf_function) = if let Some(s) = parser.get("addr") { + let (a, b) = parse_addr(s.as_str()).map_err(Error::ParseDisk)?; + (Some(a), Some(b)) + } else { + (None, None) + }; + Ok(FsConfig { tag, socket, @@ -1631,6 +1697,7 @@ impl FsConfig { queue_size, id, pci_segment, + bdf_device_id, }) } @@ -1756,7 +1823,7 @@ impl FwCfgItem { impl PmemConfig { pub const SYNTAX: &'static str = "Persistent memory parameters \ \"file=,size=,iommu=on|off,\ - discard_writes=on|off,id=,pci_segment=\""; + discard_writes=on|off,id=,pci_segment=,addr=\""; pub fn parse(pmem: &str) -> Result { let mut parser = OptionParser::new(); @@ -1766,7 +1833,8 @@ impl PmemConfig { .add("iommu") .add("discard_writes") .add("id") - .add("pci_segment"); + .add("pci_segment") + .add("addr"); parser.parse(pmem).map_err(Error::ParsePersistentMemory)?; let file = PathBuf::from(parser.get("file").ok_or(Error::ParsePmemFileMissing)?); @@ -1790,6 +1858,13 @@ impl PmemConfig { .map_err(Error::ParsePersistentMemory)? .unwrap_or_default(); + let (bdf_device_id, _bdf_function) = if let Some(s) = parser.get("addr") { + let (a, b) = parse_addr(s.as_str()).map_err(Error::ParseDisk)?; + (Some(a), Some(b)) + } else { + (None, None) + }; + Ok(PmemConfig { file, size, @@ -1797,6 +1872,7 @@ impl PmemConfig { discard_writes, id, pci_segment, + bdf_device_id, }) } @@ -1829,7 +1905,8 @@ impl ConsoleConfig { .add("file") .add("iommu") .add("tcp") - .add("socket"); + .add("socket") + .add("addr"); parser.parse(console).map_err(Error::ParseConsole)?; let mut file: Option = default_consoleconfig_file(); @@ -1877,12 +1954,20 @@ impl ConsoleConfig { .unwrap_or(Toggle(false)) .0; + let (bdf_device_id, _bdf_function) = if let Some(s) = parser.get("addr") { + let (a, b) = parse_addr(s.as_str()).map_err(Error::ParseDisk)?; + (Some(a), Some(b)) + } else { + (None, None) + }; + Ok(Self { file, mode, iommu, socket, url, + bdf_device_id, }) } } @@ -1942,7 +2027,8 @@ impl DebugConsoleConfig { } impl DeviceConfig { - pub const SYNTAX: &'static str = "Direct device assignment parameters \"path=,iommu=on|off,id=,pci_segment=\""; + pub const SYNTAX: &'static str = "Direct device assignment parameters \"\ + path=,iommu=on|off,id=,pci_segment=\""; pub fn parse(device: &str) -> Result { let mut parser = OptionParser::new(); @@ -2046,7 +2132,7 @@ impl UserDeviceConfig { impl VdpaConfig { pub const SYNTAX: &'static str = "vDPA device \ \"path=,num_queues=,iommu=on|off,\ - id=,pci_segment=\""; + id=,pci_segment=,addr=\""; pub fn parse(vdpa: &str) -> Result { let mut parser = OptionParser::new(); @@ -2055,7 +2141,8 @@ impl VdpaConfig { .add("num_queues") .add("iommu") .add("id") - .add("pci_segment"); + .add("pci_segment") + .add("addr"); parser.parse(vdpa).map_err(Error::ParseVdpa)?; let path = parser @@ -2077,12 +2164,20 @@ impl VdpaConfig { .map_err(Error::ParseVdpa)? .unwrap_or_default(); + let (bdf_device_id, _bdf_function) = if let Some(s) = parser.get("addr") { + let (a, b) = parse_addr(s.as_str()).map_err(Error::ParseDisk)?; + (Some(a), Some(b)) + } else { + (None, None) + }; + Ok(VdpaConfig { path, num_queues, iommu, id, pci_segment, + bdf_device_id, }) } @@ -2106,7 +2201,8 @@ impl VdpaConfig { impl VsockConfig { pub const SYNTAX: &'static str = "Virtio VSOCK parameters \ - \"cid=,socket=,iommu=on|off,id=,pci_segment=\""; + \"cid=,socket=,iommu=on|off,id=,\ + pci_segment=,addr=\""; pub fn parse(vsock: &str) -> Result { let mut parser = OptionParser::new(); @@ -2115,7 +2211,8 @@ impl VsockConfig { .add("cid") .add("iommu") .add("id") - .add("pci_segment"); + .add("pci_segment") + .add("addr"); parser.parse(vsock).map_err(Error::ParseVsock)?; let socket = parser @@ -2137,12 +2234,20 @@ impl VsockConfig { .map_err(Error::ParseVsock)? .unwrap_or_default(); + let (bdf_device_id, _bdf_function) = if let Some(s) = parser.get("addr") { + let (a, b) = parse_addr(s.as_str()).map_err(Error::ParseDisk)?; + (Some(a), Some(b)) + } else { + (None, None) + }; + Ok(VsockConfig { cid, socket, iommu, id, pci_segment, + bdf_device_id, }) } @@ -3453,6 +3558,7 @@ mod tests { pci_segment: 0, serial: None, queue_affinity: None, + bdf_device_id: None, } } @@ -3547,6 +3653,13 @@ mod tests { ..disk_fixture() } ); + assert_eq!( + DiskConfig::parse("path=/path/to_file,addr=15.0")?, + DiskConfig { + bdf_device_id: Some(15), + ..disk_fixture() + } + ); Ok(()) } @@ -3571,6 +3684,7 @@ mod tests { offload_tso: true, offload_ufo: true, offload_csum: true, + bdf_device_id: None, } } @@ -3635,6 +3749,14 @@ mod tests { } ); + assert_eq!( + NetConfig::parse("mac=de:ad:be:ef:12:34,host_mac=12:34:de:ad:be:ef,addr=15.0")?, + NetConfig { + bdf_device_id: Some(15), + ..net_fixture() + } + ); + Ok(()) } @@ -3653,6 +3775,7 @@ mod tests { RngConfig { src: PathBuf::from("/dev/random"), iommu: true, + bdf_device_id: None, } ); assert_eq!( @@ -3662,6 +3785,13 @@ mod tests { ..Default::default() } ); + assert_eq!( + RngConfig::parse("addr=15.0")?, + RngConfig { + bdf_device_id: Some(15), + ..Default::default() + } + ); Ok(()) } @@ -3673,6 +3803,7 @@ mod tests { queue_size: 1024, id: None, pci_segment: 0, + bdf_device_id: None, } } @@ -3692,6 +3823,14 @@ mod tests { } ); + assert_eq!( + FsConfig::parse("tag=mytag,socket=/tmp/sock,addr=15.0")?, + FsConfig { + bdf_device_id: Some(15), + ..fs_fixture() + } + ); + Ok(()) } @@ -3703,6 +3842,7 @@ mod tests { discard_writes: false, id: None, pci_segment: 0, + bdf_device_id: None, } } @@ -3730,6 +3870,13 @@ mod tests { ..pmem_fixture() } ); + assert_eq!( + PmemConfig::parse("file=/tmp/pmem,size=128M,addr=15.0")?, + PmemConfig { + bdf_device_id: Some(15), + ..pmem_fixture() + } + ); Ok(()) } @@ -3746,6 +3893,7 @@ mod tests { file: None, socket: None, url: None, + bdf_device_id: None, } ); assert_eq!( @@ -3756,6 +3904,7 @@ mod tests { file: None, socket: None, url: None, + bdf_device_id: None, } ); assert_eq!( @@ -3766,6 +3915,7 @@ mod tests { file: None, socket: None, url: None, + bdf_device_id: None, } ); assert_eq!( @@ -3776,6 +3926,7 @@ mod tests { file: None, socket: None, url: None, + bdf_device_id: None, } ); assert_eq!( @@ -3786,6 +3937,7 @@ mod tests { file: Some(PathBuf::from("/tmp/console")), socket: None, url: None, + bdf_device_id: None, } ); assert_eq!( @@ -3796,6 +3948,7 @@ mod tests { file: None, socket: None, url: None, + bdf_device_id: None, } ); assert_eq!( @@ -3806,6 +3959,7 @@ mod tests { file: Some(PathBuf::from("/tmp/console")), socket: None, url: None, + bdf_device_id: None, } ); assert_eq!( @@ -3816,6 +3970,7 @@ mod tests { file: None, socket: Some(PathBuf::from("/tmp/serial.sock")), url: None, + bdf_device_id: None, } ); Ok(()) @@ -3867,6 +4022,7 @@ mod tests { iommu: false, id: None, pci_segment: 0, + bdf_device_id: None, } } @@ -3883,6 +4039,13 @@ mod tests { ..vdpa_fixture() } ); + assert_eq!( + VdpaConfig::parse("path=/dev/vhost-vdpa,addr=15.0")?, + VdpaConfig { + bdf_device_id: Some(15), + ..vdpa_fixture() + } + ); Ok(()) } @@ -3911,6 +4074,7 @@ mod tests { iommu: false, id: None, pci_segment: 0, + bdf_device_id: None, } ); assert_eq!( @@ -3921,6 +4085,19 @@ mod tests { iommu: true, id: None, pci_segment: 0, + bdf_device_id: None, + } + ); + + assert_eq!( + VsockConfig::parse("socket=/tmp/sock,cid=3,iommu=on,addr=15.0")?, + VsockConfig { + cid: 3, + socket: PathBuf::from("/tmp/sock"), + iommu: true, + id: None, + pci_segment: 0, + bdf_device_id: Some(15), } ); Ok(()) @@ -4000,6 +4177,7 @@ mod tests { id: Some("net0".to_owned()), num_queues: 2, fds: Some(vec![-1, -1, -1, -1]), + bdf_device_id: Some(15), ..net_fixture() }, NetConfig { @@ -4173,6 +4351,7 @@ mod tests { rng: RngConfig { src: PathBuf::from("/dev/urandom"), iommu: false, + bdf_device_id: None, }, balloon: None, fs: None, @@ -4183,6 +4362,7 @@ mod tests { iommu: false, socket: None, url: None, + bdf_device_id: None, }, console: ConsoleConfig { file: None, @@ -4190,6 +4370,7 @@ mod tests { iommu: false, socket: None, url: None, + bdf_device_id: None, }, #[cfg(target_arch = "x86_64")] debug_console: DebugConsoleConfig::default(), @@ -4485,6 +4666,7 @@ mod tests { id: None, iommu: true, pci_segment: 1, + bdf_device_id: None, }); still_valid_config.validate().unwrap(); @@ -4561,6 +4743,7 @@ mod tests { id: None, iommu: false, pci_segment: 1, + bdf_device_id: None, }); assert_eq!( invalid_config.validate(), diff --git a/vmm/src/device_manager.rs b/vmm/src/device_manager.rs index fa409a27e8..1f8da19a2f 100644 --- a/vmm/src/device_manager.rs +++ b/vmm/src/device_manager.rs @@ -9,7 +9,7 @@ // SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause // -use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::collections::{BTreeMap, BTreeSet, HashMap, VecDeque}; use std::fs::{File, OpenOptions}; use std::io::{self, IsTerminal, Seek, SeekFrom, stdout}; use std::num::Wrapping; @@ -467,7 +467,7 @@ pub enum DeviceManagerError { /// Failed to find an available PCI device ID. #[error("Failed to find an available PCI device ID")] - NextPciDeviceId(#[source] pci::PciRootError), + AllocatePciDeviceId(#[source] pci::PciRootError), /// Could not reserve the PCI device ID. #[error("Could not reserve the PCI device ID")] @@ -896,6 +896,7 @@ struct MetaVirtioDevice { iommu: bool, id: String, pci_segment: u16, + bdf_device: Option, dma_handler: Option>, } @@ -981,7 +982,7 @@ pub struct DeviceManager { cpu_manager: Arc>, // The virtio devices on the system - virtio_devices: Vec, + virtio_devices: VecDeque, /// All disks. Needed for locking and unlocking the images. block_devices: Vec>>, @@ -1320,7 +1321,7 @@ impl DeviceManager { config, memory_manager, cpu_manager, - virtio_devices: Vec::new(), + virtio_devices: VecDeque::new(), block_devices: vec![], bus_devices: Vec::new(), device_id_cnt, @@ -1406,8 +1407,6 @@ impl DeviceManager { ) -> DeviceManagerResult<()> { trace_scoped!("create_devices"); - let mut virtio_devices: Vec = Vec::new(); - self.cpu_manager .lock() .unwrap() @@ -1458,12 +1457,8 @@ impl DeviceManager { self.original_termios_opt = original_termios_opt; - self.console = self.add_console_devices( - &legacy_interrupt_manager, - &mut virtio_devices, - console_info, - console_resize_pipe, - )?; + self.console = + self.add_console_devices(&legacy_interrupt_manager, console_info, console_resize_pipe)?; #[cfg(not(target_arch = "riscv64"))] if let Some(tpm) = self.config.clone().lock().unwrap().tpm.as_ref() { @@ -1473,11 +1468,8 @@ impl DeviceManager { } self.legacy_interrupt_manager = Some(legacy_interrupt_manager); - virtio_devices.append(&mut self.make_virtio_devices()?); - - self.add_pci_devices(virtio_devices.clone())?; - - self.virtio_devices = virtio_devices; + self.make_virtio_devices()?; + self.add_pci_devices()?; // Add pvmemcontrol if required #[cfg(feature = "pvmemcontrol")] @@ -1586,10 +1578,7 @@ impl DeviceManager { } #[allow(unused_variables)] - fn add_pci_devices( - &mut self, - virtio_devices: Vec, - ) -> DeviceManagerResult<()> { + fn add_pci_devices(&mut self) -> DeviceManagerResult<()> { let iommu_id = String::from(IOMMU_DEVICE_NAME); let iommu_address_width_bits = @@ -1631,7 +1620,7 @@ impl DeviceManager { let mut iommu_attached_devices = Vec::new(); { - for handle in virtio_devices { + for handle in self.virtio_devices.clone() { let mapping: Option> = if handle.iommu { self.iommu_mapping.clone() } else { @@ -1644,6 +1633,7 @@ impl DeviceManager { handle.id, handle.pci_segment, handle.dma_handler, + handle.bdf_device, )?; if handle.iommu { @@ -1672,7 +1662,8 @@ impl DeviceManager { } if let Some(iommu_device) = iommu_device { - let dev_id = self.add_virtio_pci_device(iommu_device, &None, iommu_id, 0, None)?; + let dev_id = + self.add_virtio_pci_device(iommu_device, &None, iommu_id, 0, None, None)?; self.iommu_attached_devices = Some((dev_id, iommu_attached_devices)); } } @@ -2319,7 +2310,6 @@ impl DeviceManager { fn add_virtio_console_device( &mut self, - virtio_devices: &mut Vec, console_fd: ConsoleOutput, resize_pipe: Option>, ) -> DeviceManagerResult>> { @@ -2378,14 +2368,21 @@ impl DeviceManager { ) .map_err(DeviceManagerError::CreateVirtioConsole)?; let virtio_console_device = Arc::new(Mutex::new(virtio_console_device)); - virtio_devices.push(MetaVirtioDevice { + let device = MetaVirtioDevice { virtio_device: Arc::clone(&virtio_console_device) as Arc>, iommu: console_config.iommu, id: id.clone(), pci_segment: 0, dma_handler: None, - }); + bdf_device: console_config.bdf_device_id, + }; + + if console_config.bdf_device_id.is_some() { + self.virtio_devices.push_front(device); + } else { + self.virtio_devices.push_back(device); + } // Fill the device tree with a new node. In case of restore, we // know there is nothing to do, so we can simply override the @@ -2411,7 +2408,6 @@ impl DeviceManager { fn add_console_devices( &mut self, interrupt_manager: &Arc>, - virtio_devices: &mut Vec, console_info: Option, console_resize_pipe: Option>, ) -> DeviceManagerResult> { @@ -2480,11 +2476,8 @@ impl DeviceManager { } } - let console_resizer = self.add_virtio_console_device( - virtio_devices, - console_info.console_main_fd, - console_resize_pipe, - )?; + let console_resizer = + self.add_virtio_console_device(console_info.console_main_fd, console_resize_pipe)?; Ok(Arc::new(Console { console_resizer })) } @@ -2542,35 +2535,33 @@ impl DeviceManager { Ok(()) } - fn make_virtio_devices(&mut self) -> DeviceManagerResult> { - let mut devices: Vec = Vec::new(); - + fn make_virtio_devices(&mut self) -> DeviceManagerResult<()> { // Create "standard" virtio devices (net/block/rng) - devices.append(&mut self.make_virtio_block_devices()?); - devices.append(&mut self.make_virtio_net_devices()?); - devices.append(&mut self.make_virtio_rng_devices()?); + self.make_virtio_block_devices()?; + self.make_virtio_net_devices()?; + self.make_virtio_rng_devices()?; // Add virtio-fs if required - devices.append(&mut self.make_virtio_fs_devices()?); + self.make_virtio_fs_devices()?; // Add virtio-pmem if required - devices.append(&mut self.make_virtio_pmem_devices()?); + self.make_virtio_pmem_devices()?; // Add virtio-vsock if required - devices.append(&mut self.make_virtio_vsock_devices()?); + self.make_virtio_vsock_devices()?; - devices.append(&mut self.make_virtio_mem_devices()?); + self.make_virtio_mem_devices()?; // Add virtio-balloon if required - devices.append(&mut self.make_virtio_balloon_devices()?); + self.make_virtio_balloon_devices()?; // Add virtio-watchdog device - devices.append(&mut self.make_virtio_watchdog_devices()?); + self.make_virtio_watchdog_devices()?; // Add vDPA devices if required - devices.append(&mut self.make_vdpa_devices()?); + self.make_vdpa_devices()?; - Ok(devices) + Ok(()) } // Cache whether aio is supported to avoid checking for very block device @@ -2838,21 +2829,25 @@ impl DeviceManager { id, pci_segment: disk_cfg.pci_segment, dma_handler: None, + bdf_device: disk_cfg.bdf_device_id, }) } - fn make_virtio_block_devices(&mut self) -> DeviceManagerResult> { - let mut devices = Vec::new(); - + fn make_virtio_block_devices(&mut self) -> DeviceManagerResult<()> { let mut block_devices = self.config.lock().unwrap().disks.clone(); if let Some(disk_list_cfg) = &mut block_devices { for disk_cfg in disk_list_cfg.iter_mut() { - devices.push(self.make_virtio_block_device(disk_cfg, false)?); + let device = self.make_virtio_block_device(disk_cfg, false)?; + if disk_cfg.bdf_device_id.is_some() { + self.virtio_devices.push_front(device); + } else { + self.virtio_devices.push_back(device); + } } } self.config.lock().unwrap().disks = block_devices; - Ok(devices) + Ok(()) } fn make_virtio_net_device( @@ -3015,26 +3010,29 @@ impl DeviceManager { id, pci_segment: net_cfg.pci_segment, dma_handler: None, + bdf_device: net_cfg.bdf_device_id, }) } /// Add virto-net and vhost-user-net devices - fn make_virtio_net_devices(&mut self) -> DeviceManagerResult> { - let mut devices = Vec::new(); + fn make_virtio_net_devices(&mut self) -> DeviceManagerResult<()> { let mut net_devices = self.config.lock().unwrap().net.clone(); if let Some(net_list_cfg) = &mut net_devices { for net_cfg in net_list_cfg.iter_mut() { - devices.push(self.make_virtio_net_device(net_cfg)?); + let device = self.make_virtio_net_device(net_cfg)?; + if net_cfg.bdf_device_id.is_some() { + self.virtio_devices.push_front(device); + } else { + self.virtio_devices.push_back(device); + } } } self.config.lock().unwrap().net = net_devices; - Ok(devices) + Ok(()) } - fn make_virtio_rng_devices(&mut self) -> DeviceManagerResult> { - let mut devices = Vec::new(); - + fn make_virtio_rng_devices(&mut self) -> DeviceManagerResult<()> { // Add virtio-rng if required let rng_config = self.config.lock().unwrap().rng.clone(); if let Some(rng_path) = rng_config.src.to_str() { @@ -3055,14 +3053,20 @@ impl DeviceManager { ) .map_err(DeviceManagerError::CreateVirtioRng)?, )); - devices.push(MetaVirtioDevice { + let device = MetaVirtioDevice { virtio_device: Arc::clone(&virtio_rng_device) as Arc>, iommu: rng_config.iommu, id: id.clone(), pci_segment: 0, dma_handler: None, - }); + bdf_device: rng_config.bdf_device_id, + }; + if rng_config.bdf_device_id.is_some() { + self.virtio_devices.push_front(device); + } else { + self.virtio_devices.push_back(device); + } // Fill the device tree with a new node. In case of restore, we // know there is nothing to do, so we can simply override the @@ -3073,7 +3077,7 @@ impl DeviceManager { .insert(id.clone(), device_node!(id, virtio_rng_device)); } - Ok(devices) + Ok(()) } fn make_virtio_fs_device( @@ -3123,24 +3127,28 @@ impl DeviceManager { id, pci_segment: fs_cfg.pci_segment, dma_handler: None, + bdf_device: fs_cfg.bdf_device_id, }) } else { Err(DeviceManagerError::NoVirtioFsSock) } } - fn make_virtio_fs_devices(&mut self) -> DeviceManagerResult> { - let mut devices = Vec::new(); - + fn make_virtio_fs_devices(&mut self) -> DeviceManagerResult<()> { let mut fs_devices = self.config.lock().unwrap().fs.clone(); if let Some(fs_list_cfg) = &mut fs_devices { for fs_cfg in fs_list_cfg.iter_mut() { - devices.push(self.make_virtio_fs_device(fs_cfg)?); + let device = self.make_virtio_fs_device(fs_cfg)?; + if fs_cfg.bdf_device_id.is_some() { + self.virtio_devices.push_front(device); + } else { + self.virtio_devices.push_back(device); + } } } self.config.lock().unwrap().fs = fs_devices; - Ok(devices) + Ok(()) } fn make_virtio_pmem_device( @@ -3312,21 +3320,26 @@ impl DeviceManager { id, pci_segment: pmem_cfg.pci_segment, dma_handler: None, + bdf_device: pmem_cfg.bdf_device_id, }) } - fn make_virtio_pmem_devices(&mut self) -> DeviceManagerResult> { - let mut devices = Vec::new(); + fn make_virtio_pmem_devices(&mut self) -> DeviceManagerResult<()> { // Add virtio-pmem if required let mut pmem_devices = self.config.lock().unwrap().pmem.clone(); if let Some(pmem_list_cfg) = &mut pmem_devices { for pmem_cfg in pmem_list_cfg.iter_mut() { - devices.push(self.make_virtio_pmem_device(pmem_cfg)?); + let device = self.make_virtio_pmem_device(pmem_cfg)?; + if pmem_cfg.bdf_device_id.is_some() { + self.virtio_devices.push_front(device); + } else { + self.virtio_devices.push_back(device); + } } } self.config.lock().unwrap().pmem = pmem_devices; - Ok(devices) + Ok(()) } fn make_virtio_vsock_device( @@ -3383,24 +3396,26 @@ impl DeviceManager { id, pci_segment: vsock_cfg.pci_segment, dma_handler: None, + bdf_device: vsock_cfg.bdf_device_id, }) } - fn make_virtio_vsock_devices(&mut self) -> DeviceManagerResult> { - let mut devices = Vec::new(); - + fn make_virtio_vsock_devices(&mut self) -> DeviceManagerResult<()> { let mut vsock = self.config.lock().unwrap().vsock.clone(); if let Some(vsock_cfg) = &mut vsock { - devices.push(self.make_virtio_vsock_device(vsock_cfg)?); + let device = self.make_virtio_vsock_device(vsock_cfg)?; + if vsock_cfg.bdf_device_id.is_some() { + self.virtio_devices.push_front(device); + } else { + self.virtio_devices.push_back(device); + } } self.config.lock().unwrap().vsock = vsock; - Ok(devices) + Ok(()) } - fn make_virtio_mem_devices(&mut self) -> DeviceManagerResult> { - let mut devices = Vec::new(); - + fn make_virtio_mem_devices(&mut self) -> DeviceManagerResult<()> { let mm = self.memory_manager.clone(); let mut mm = mm.lock().unwrap(); for (memory_zone_id, memory_zone) in mm.memory_zones_mut().iter_mut() { @@ -3435,13 +3450,14 @@ impl DeviceManager { self.virtio_mem_devices.push(Arc::clone(&virtio_mem_device)); - devices.push(MetaVirtioDevice { + self.virtio_devices.push_back(MetaVirtioDevice { virtio_device: Arc::clone(&virtio_mem_device) as Arc>, iommu: false, id: memory_zone_id.clone(), pci_segment: 0, dma_handler: None, + bdf_device: None, }); // Fill the device tree with a new node. In case of restore, we @@ -3454,7 +3470,7 @@ impl DeviceManager { } } - Ok(devices) + Ok(()) } #[cfg(feature = "pvmemcontrol")] @@ -3468,7 +3484,7 @@ impl DeviceManager { let pci_segment_id = 0x0_u16; let (pci_segment_id, pci_device_bdf, resources) = - self.pci_resources(&id, pci_segment_id)?; + self.pci_resources(&id, pci_segment_id, None)?; info!("Creating pvmemcontrol device: id = {}", id); let (pvmemcontrol_pci_device, pvmemcontrol_bus_device) = @@ -3499,9 +3515,7 @@ impl DeviceManager { Ok((pvmemcontrol_bus_device, pvmemcontrol_pci_device)) } - fn make_virtio_balloon_devices(&mut self) -> DeviceManagerResult> { - let mut devices = Vec::new(); - + fn make_virtio_balloon_devices(&mut self) -> DeviceManagerResult<()> { if let Some(balloon_config) = &self.config.lock().unwrap().balloon { let id = String::from(BALLOON_DEVICE_NAME); info!("Creating virtio-balloon device: id = {}", id); @@ -3524,14 +3538,21 @@ impl DeviceManager { self.balloon = Some(virtio_balloon_device.clone()); - devices.push(MetaVirtioDevice { + let device = MetaVirtioDevice { virtio_device: Arc::clone(&virtio_balloon_device) as Arc>, iommu: false, id: id.clone(), pci_segment: 0, dma_handler: None, - }); + bdf_device: balloon_config.bdf_device_id, + }; + + if balloon_config.bdf_device_id.is_some() { + self.virtio_devices.push_front(device); + } else { + self.virtio_devices.push_back(device); + } self.device_tree .lock() @@ -3539,14 +3560,12 @@ impl DeviceManager { .insert(id.clone(), device_node!(id, virtio_balloon_device)); } - Ok(devices) + Ok(()) } - fn make_virtio_watchdog_devices(&mut self) -> DeviceManagerResult> { - let mut devices = Vec::new(); - + fn make_virtio_watchdog_devices(&mut self) -> DeviceManagerResult<()> { if !self.config.lock().unwrap().watchdog { - return Ok(devices); + return Ok(()); } let id = String::from(WATCHDOG_DEVICE_NAME); @@ -3565,13 +3584,14 @@ impl DeviceManager { ) .map_err(DeviceManagerError::CreateVirtioWatchdog)?, )); - devices.push(MetaVirtioDevice { + self.virtio_devices.push_back(MetaVirtioDevice { virtio_device: Arc::clone(&virtio_watchdog_device) as Arc>, iommu: false, id: id.clone(), pci_segment: 0, dma_handler: None, + bdf_device: None, }); self.device_tree @@ -3579,7 +3599,7 @@ impl DeviceManager { .unwrap() .insert(id.clone(), device_node!(id, virtio_watchdog_device)); - Ok(devices) + Ok(()) } fn make_vdpa_device( @@ -3630,21 +3650,26 @@ impl DeviceManager { id, pci_segment: vdpa_cfg.pci_segment, dma_handler: Some(vdpa_mapping), + bdf_device: vdpa_cfg.bdf_device_id, }) } - fn make_vdpa_devices(&mut self) -> DeviceManagerResult> { - let mut devices = Vec::new(); + fn make_vdpa_devices(&mut self) -> DeviceManagerResult<()> { // Add vdpa if required let mut vdpa_devices = self.config.lock().unwrap().vdpa.clone(); if let Some(vdpa_list_cfg) = &mut vdpa_devices { for vdpa_cfg in vdpa_list_cfg.iter_mut() { - devices.push(self.make_vdpa_device(vdpa_cfg)?); + let device = self.make_vdpa_device(vdpa_cfg)?; + if vdpa_cfg.bdf_device_id.is_some() { + self.virtio_devices.push_front(device); + } else { + self.virtio_devices.push_back(device); + } } } self.config.lock().unwrap().vdpa = vdpa_devices; - Ok(devices) + Ok(()) } fn next_device_name(&mut self, prefix: &str) -> DeviceManagerResult { @@ -3716,7 +3741,7 @@ impl DeviceManager { }; let (pci_segment_id, pci_device_bdf, resources) = - self.pci_resources(&vfio_name, device_cfg.pci_segment)?; + self.pci_resources(&vfio_name, device_cfg.pci_segment, None)?; let mut needs_dma_mapping = false; @@ -3953,7 +3978,7 @@ impl DeviceManager { }; let (pci_segment_id, pci_device_bdf, resources) = - self.pci_resources(&vfio_user_name, device_cfg.pci_segment)?; + self.pci_resources(&vfio_user_name, device_cfg.pci_segment, None)?; let legacy_interrupt_group = if let Some(legacy_interrupt_manager) = &self.legacy_interrupt_manager { @@ -4065,6 +4090,7 @@ impl DeviceManager { virtio_device_id: String, pci_segment_id: u16, dma_handler: Option>, + bdf_device: Option, ) -> DeviceManagerResult { let id = format!("{VIRTIO_PCI_DEVICE_NAME_PREFIX}-{virtio_device_id}"); @@ -4073,7 +4099,7 @@ impl DeviceManager { node.children = vec![virtio_device_id.clone()]; let (pci_segment_id, pci_device_bdf, resources) = - self.pci_resources(&id, pci_segment_id)?; + self.pci_resources(&id, pci_segment_id, bdf_device)?; // Update the existing virtio node by setting the parent. if let Some(node) = self.device_tree.lock().unwrap().get_mut(&virtio_device_id) { @@ -4210,7 +4236,7 @@ impl DeviceManager { info!("Creating pvpanic device {}", id); let (pci_segment_id, pci_device_bdf, resources) = - self.pci_resources(&id, pci_segment_id)?; + self.pci_resources(&id, pci_segment_id, None)?; let snapshot = snapshot_from_id(self.snapshot.as_ref(), id.as_str()); @@ -4248,7 +4274,7 @@ impl DeviceManager { info!("Creating ivshmem device {}", id); let (pci_segment_id, pci_device_bdf, resources) = - self.pci_resources(&id, pci_segment_id)?; + self.pci_resources(&id, pci_segment_id, None)?; let snapshot = snapshot_from_id(self.snapshot.as_ref(), id.as_str()); let ivshmem_ops = Arc::new(Mutex::new(IvshmemHandler { @@ -4293,6 +4319,7 @@ impl DeviceManager { &self, id: &str, pci_segment_id: u16, + pci_device_id: Option, ) -> DeviceManagerResult<(u16, PciBdf, Option>)> { // Look for the id in the device tree. If it can be found, that means // the device is being restored, otherwise it's created from scratch. @@ -4319,7 +4346,8 @@ impl DeviceManager { (pci_segment_id, pci_device_bdf, resources) } else { - let pci_device_bdf = self.pci_segments[pci_segment_id as usize].next_device_bdf()?; + let pci_device_bdf = + self.pci_segments[pci_segment_id as usize].allocate_device_bdf(pci_device_id)?; (pci_segment_id, pci_device_bdf, None) }) @@ -4785,7 +4813,7 @@ impl DeviceManager { // Add the virtio device to the device manager list. This is important // as the list is used to notify virtio devices about memory updates // for instance. - self.virtio_devices.push(handle.clone()); + self.virtio_devices.push_back(handle.clone()); let mapping: Option> = if handle.iommu { self.iommu_mapping.clone() @@ -4799,6 +4827,7 @@ impl DeviceManager { handle.id.clone(), handle.pci_segment, handle.dma_handler, + handle.bdf_device, )?; // Update the PCIU bitmap diff --git a/vmm/src/lib.rs b/vmm/src/lib.rs index 7ec33a23df..06e7c4ed56 100644 --- a/vmm/src/lib.rs +++ b/vmm/src/lib.rs @@ -3475,6 +3475,7 @@ mod unit_tests { rng: RngConfig { src: PathBuf::from("/dev/urandom"), iommu: false, + bdf_device_id: None, }, balloon: None, fs: None, @@ -3485,6 +3486,7 @@ mod unit_tests { iommu: false, socket: None, url: None, + bdf_device_id: None, }, console: ConsoleConfig { file: None, @@ -3493,6 +3495,7 @@ mod unit_tests { iommu: false, socket: None, url: None, + bdf_device_id: None, }, #[cfg(target_arch = "x86_64")] debug_console: DebugConsoleConfig::default(), diff --git a/vmm/src/pci_segment.rs b/vmm/src/pci_segment.rs index 345869c1da..8cb3193139 100644 --- a/vmm/src/pci_segment.rs +++ b/vmm/src/pci_segment.rs @@ -163,15 +163,22 @@ impl PciSegment { ) } - pub(crate) fn next_device_bdf(&self) -> DeviceManagerResult { + /// Allocates a device BDF on this PCI segment. + /// + /// - `device_id`: Device ID to request for BDF allocation + /// + /// ## Errors + /// * [`DeviceManagerError::AllocatePciDeviceId`] if device ID + /// allocation on the bus fails. + pub(crate) fn allocate_device_bdf(&self, device_id: Option) -> DeviceManagerResult { Ok(PciBdf::new( self.id, 0, self.pci_bus .lock() .unwrap() - .next_device_id() - .map_err(DeviceManagerError::NextPciDeviceId)? as u8, + .allocate_device_id(device_id) + .map_err(DeviceManagerError::AllocatePciDeviceId)? as u8, 0, )) } @@ -201,6 +208,65 @@ impl PciSegment { Ok(()) } + + #[cfg(test)] + /// Creates a PciSegment without the need for an [`AddressManager`] + /// for testing purpose. + /// + /// An [`AddressManager`] would otherwise be required to create + /// [`PciBus`] instances. Instead, we use any struct that implements + /// [`DeviceRelocation`] to instantiate a [`PciBus`]. + pub(crate) fn new_without_address_manager( + id: u16, + numa_node: u32, + mem32_allocator: Arc>, + mem64_allocator: Arc>, + pci_irq_slots: &[u8; 32], + device_reloc: Arc, + ) -> DeviceManagerResult { + let pci_root = PciRoot::new(None); + let pci_bus = Arc::new(Mutex::new(PciBus::new(pci_root, device_reloc.clone()))); + + let pci_config_mmio = Arc::new(Mutex::new(PciConfigMmio::new(Arc::clone(&pci_bus)))); + let mmio_config_address = + layout::PCI_MMCONFIG_START.0 + layout::PCI_MMIO_CONFIG_SIZE_PER_SEGMENT * id as u64; + + let start_of_mem32_area = mem32_allocator.lock().unwrap().base().0; + let end_of_mem32_area = mem32_allocator.lock().unwrap().end().0; + + let start_of_mem64_area = mem64_allocator.lock().unwrap().base().0; + let end_of_mem64_area = mem64_allocator.lock().unwrap().end().0; + + let segment = PciSegment { + id, + pci_bus, + pci_config_mmio, + mmio_config_address, + proximity_domain: numa_node, + pci_devices_up: 0, + pci_devices_down: 0, + #[cfg(target_arch = "x86_64")] + pci_config_io: None, + mem32_allocator, + mem64_allocator, + start_of_mem32_area, + end_of_mem32_area, + start_of_mem64_area, + end_of_mem64_area, + pci_irq_slots: *pci_irq_slots, + }; + + info!( + "Adding PCI segment: id={}, PCI MMIO config address: 0x{:x}, mem32 area [0x{:x}-0x{:x}, mem64 area [0x{:x}-0x{:x}", + segment.id, + segment.mmio_config_address, + segment.start_of_mem32_area, + segment.end_of_mem32_area, + segment.start_of_mem64_area, + segment.end_of_mem64_area + ); + Ok(segment) + } } struct PciDevSlot { @@ -473,3 +539,96 @@ impl Aml for PciSegment { .to_aml_bytes(sink) } } + +#[cfg(test)] +mod tests { + use std::result::Result; + + use vm_memory::GuestAddress; + + use super::*; + + #[derive(Debug)] + struct MocRelocDevice; + impl DeviceRelocation for MocRelocDevice { + fn move_bar( + &self, + _old_base: u64, + _new_base: u64, + _len: u64, + _pci_dev: &mut dyn pci::PciDevice, + _region_type: pci::PciBarRegionType, + ) -> Result<(), std::io::Error> { + Ok(()) + } + } + + fn setup() -> PciSegment { + let guest_addr = 0_u64; + let guest_size = 0x1000_usize; + let allocator_1 = Arc::new(Mutex::new( + AddressAllocator::new(GuestAddress(guest_addr), guest_size as u64).unwrap(), + )); + let allocator_2 = Arc::new(Mutex::new( + AddressAllocator::new(GuestAddress(guest_addr), guest_size as u64).unwrap(), + )); + let moc_device_reloc = Arc::new(MocRelocDevice {}); + let arr = [0_u8; 32]; + + PciSegment::new_without_address_manager( + 0, + 0, + allocator_1, + allocator_2, + &arr, + moc_device_reloc, + ) + .unwrap() + } + + #[test] + // Test the default bdf for an an segment with an empty bus (except for the root device) + fn allocate_device_bdf_default() { + // The first address is occupied by the root + let segment = setup(); + let bdf = segment.allocate_device_bdf(None).unwrap(); + assert_eq!(bdf.segment(), segment.id); + assert_eq!(bdf.bus(), 0); + assert_eq!(bdf.device(), 1); + assert_eq!(bdf.function(), 0); + } + + #[test] + // Test to acquire a bdf with s specific device ID + fn allocate_device_bdf_fixed_device_id() { + // The first address is occupied by the root + let expect_device_id = 0x10_u8; + let segment = setup(); + let bdf = segment.allocate_device_bdf(Some(expect_device_id)).unwrap(); + assert_eq!(bdf.segment(), segment.id); + assert_eq!(bdf.bus(), 0); + assert_eq!(bdf.device(), expect_device_id); + assert_eq!(bdf.function(), 0); + } + + #[test] + // Test to acquire a bdf with invalid device id, one already + // taken and the other being greater then the number of allowed + // devices per bus. + fn allocate_device_bdf_invalid_device_id() { + // The first address is occupied by the root + let already_taken_device_id = 0x0_u8; + let overflow_device_id = 0xff_u8; + let segment = setup(); + let bdf_res = segment.allocate_device_bdf(Some(already_taken_device_id)); + assert!(matches!( + bdf_res, + Err(DeviceManagerError::AllocatePciDeviceId(_)) + )); + let bdf_res = segment.allocate_device_bdf(Some(overflow_device_id)); + assert!(matches!( + bdf_res, + Err(DeviceManagerError::AllocatePciDeviceId(_)) + )); + } +} diff --git a/vmm/src/vm_config.rs b/vmm/src/vm_config.rs index d7062e29ea..b5020342f6 100644 --- a/vmm/src/vm_config.rs +++ b/vmm/src/vm_config.rs @@ -276,6 +276,10 @@ pub struct DiskConfig { pub serial: Option, #[serde(default)] pub queue_affinity: Option>, + // #[serde(default)] + // pub bus_hint: Option, + #[serde(default)] + pub bdf_device_id: Option, } impl ApplyLandlock for DiskConfig { @@ -341,6 +345,7 @@ pub struct NetConfig { pub offload_ufo: bool, #[serde(default = "default_netconfig_true")] pub offload_csum: bool, + pub bdf_device_id: Option, } pub fn default_netconfig_true() -> bool { @@ -401,6 +406,7 @@ pub struct RngConfig { pub src: PathBuf, #[serde(default)] pub iommu: bool, + pub bdf_device_id: Option, } pub const DEFAULT_RNG_SOURCE: &str = "/dev/urandom"; @@ -410,6 +416,7 @@ impl Default for RngConfig { RngConfig { src: PathBuf::from(DEFAULT_RNG_SOURCE), iommu: false, + bdf_device_id: None, } } } @@ -431,6 +438,7 @@ pub struct BalloonConfig { /// Option to enable free page reporting from the guest. #[serde(default)] pub free_page_reporting: bool, + pub bdf_device_id: Option, } #[cfg(feature = "pvmemcontrol")] @@ -449,6 +457,7 @@ pub struct FsConfig { pub id: Option, #[serde(default)] pub pci_segment: u16, + pub bdf_device_id: Option, } pub fn default_fsconfig_num_queues() -> usize { @@ -479,6 +488,7 @@ pub struct PmemConfig { pub id: Option, #[serde(default)] pub pci_segment: u16, + pub bdf_device_id: Option, } impl ApplyLandlock for PmemConfig { @@ -509,6 +519,8 @@ pub struct ConsoleConfig { pub iommu: bool, pub socket: Option, pub url: Option, + /// PCI BDF to attach the console in the guest to + pub bdf_device_id: Option, } pub fn default_consoleconfig_file() -> Option { @@ -614,6 +626,7 @@ pub struct VdpaConfig { pub id: Option, #[serde(default)] pub pci_segment: u16, + pub bdf_device_id: Option, } pub fn default_vdpaconfig_num_queues() -> usize { @@ -637,6 +650,7 @@ pub struct VsockConfig { pub id: Option, #[serde(default)] pub pci_segment: u16, + pub bdf_device_id: Option, } impl ApplyLandlock for VsockConfig { @@ -859,6 +873,7 @@ pub fn default_serial() -> ConsoleConfig { iommu: false, socket: None, url: None, + bdf_device_id: None, } } @@ -869,6 +884,7 @@ pub fn default_console() -> ConsoleConfig { iommu: false, socket: None, url: None, + bdf_device_id: None, } }