Skip to content

Commit

Permalink
Merge pull request #8724 from openanolis/chao/add_vfio
Browse files Browse the repository at this point in the history 
dragonball: introduce vfio support
  • Loading branch information
@studychao
studychao committed Dec 27, 2023
2 parents 43a627c + 2f797a6 commit 8895cb8
Show file tree
Hide file tree
Showing 7 changed files with 2,044 additions and 58 deletions.
13 changes: 10 additions & 3 deletions src/dragonball/src/dbs_pci/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,14 +11,21 @@ keywords = ["dragonball", "secure-sandbox", "devices", "pci"]
readme = "README.md"

[dependencies]
log = "0.4.14"
thiserror = "1"
dbs-allocator = { path = "../dbs_allocator" }
dbs-boot = { path = "../dbs_boot" }
dbs-device = { path = "../dbs_device" }
dbs-interrupt = { path = "../dbs_interrupt", features = ["kvm-irq", "kvm-legacy-irq", "kvm-msi-irq"] }
dbs-allocator = { path = "../dbs_allocator" }
log = "0.4.14"
downcast-rs = "1.2.0"
byteorder = "1.4.3"
thiserror = "1"
vm-memory = "0.10.0"
kvm-ioctls = "0.12.0"
kvm-bindings = "0.6.0"
vfio-ioctls = "0.1.0"
vfio-bindings = "0.3.0"
libc = "0.2.39"

[dev-dependencies]
dbs-arch = { path = "../dbs_arch" }
kvm-ioctls = "0.12.0"
8 changes: 7 additions & 1 deletion src/dragonball/src/dbs_pci/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,4 +18,10 @@ There are several components in `dbs-pci` crate building together to emulate PCI

6. `msi` mod: struct to maintain information for PCI Message Signalled Interrupt Capability. It will be initialized when parsing PCI configuration space and used when getting interrupt capabilities.

7. `msix` mod: struct to maintain information for PCI Message Signalled Interrupt Extended Capability. It will be initialized when parsing PCI configuration space and used when getting interrupt capabilities.
7. `msix` mod: struct to maintain information for PCI Message Signalled Interrupt Extended Capability. It will be initialized when parsing PCI configuration space and used when getting interrupt capabilities.

8. `vfio` mod: `vfio` mod collects lots of information related to the `vfio` operations.
a. `vfio` `msi` and `msix` capability and state
b. `vfio` interrupt information
c. PCI region information
d. `vfio` PCI device information and state
88 changes: 44 additions & 44 deletions src/dragonball/src/dbs_pci/src/configuration.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -212,7 +212,7 @@ pub trait PciProgrammingInterface {
/// Types of PCI capabilities.
#[repr(u8)]
#[derive(PartialEq, Copy, Clone)]
pub enum PciCapabilityID {
pub enum PciCapabilityId {
ListID = 0,
PowerManagement = 0x01,
AcceleratedGraphicsPort = 0x02,
Expand All @@ -238,33 +238,33 @@ pub enum PciCapabilityID {
Test = 0xFF,
}

impl From<u8> for PciCapabilityID {
impl From<u8> for PciCapabilityId {
fn from(c: u8) -> Self {
match c {
0 => PciCapabilityID::ListID,
0x01 => PciCapabilityID::PowerManagement,
0x02 => PciCapabilityID::AcceleratedGraphicsPort,
0x03 => PciCapabilityID::VitalProductData,
0x04 => PciCapabilityID::SlotIdentification,
0x05 => PciCapabilityID::MessageSignalledInterrupts,
0x06 => PciCapabilityID::CompactPCIHotSwap,
0x07 => PciCapabilityID::PCIX,
0x08 => PciCapabilityID::HyperTransport,
0x09 => PciCapabilityID::VendorSpecific,
0x0A => PciCapabilityID::Debugport,
0x0B => PciCapabilityID::CompactPCICentralResourceControl,
0x0C => PciCapabilityID::PCIStandardHotPlugController,
0x0D => PciCapabilityID::BridgeSubsystemVendorDeviceID,
0x0E => PciCapabilityID::AGPTargetPCIPCIbridge,
0x0F => PciCapabilityID::SecureDevice,
0x10 => PciCapabilityID::PCIExpress,
0x11 => PciCapabilityID::MSIX,
0x12 => PciCapabilityID::SATADataIndexConf,
0x13 => PciCapabilityID::PCIAdvancedFeatures,
0x14 => PciCapabilityID::PCIEnhancedAllocation,
0 => PciCapabilityId::ListID,
0x01 => PciCapabilityId::PowerManagement,
0x02 => PciCapabilityId::AcceleratedGraphicsPort,
0x03 => PciCapabilityId::VitalProductData,
0x04 => PciCapabilityId::SlotIdentification,
0x05 => PciCapabilityId::MessageSignalledInterrupts,
0x06 => PciCapabilityId::CompactPCIHotSwap,
0x07 => PciCapabilityId::PCIX,
0x08 => PciCapabilityId::HyperTransport,
0x09 => PciCapabilityId::VendorSpecific,
0x0A => PciCapabilityId::Debugport,
0x0B => PciCapabilityId::CompactPCICentralResourceControl,
0x0C => PciCapabilityId::PCIStandardHotPlugController,
0x0D => PciCapabilityId::BridgeSubsystemVendorDeviceID,
0x0E => PciCapabilityId::AGPTargetPCIPCIbridge,
0x0F => PciCapabilityId::SecureDevice,
0x10 => PciCapabilityId::PCIExpress,
0x11 => PciCapabilityId::MSIX,
0x12 => PciCapabilityId::SATADataIndexConf,
0x13 => PciCapabilityId::PCIAdvancedFeatures,
0x14 => PciCapabilityId::PCIEnhancedAllocation,
#[cfg(test)]
0xFF => PciCapabilityID::Test,
_ => PciCapabilityID::ListID,
0xFF => PciCapabilityId::Test,
_ => PciCapabilityId::ListID,
}
}
}
Expand Down Expand Up @@ -309,7 +309,7 @@ pub trait PciCapability: Send + Sync {
}

/// The type of PCI Interrupt
fn pci_capability_type(&self) -> PciCapabilityID;
fn pci_capability_type(&self) -> PciCapabilityId;
}

/// PCI device has four interrupt pins A->D.
Expand All @@ -334,7 +334,7 @@ pub enum PciBarRegionType {
/// 32-bit MMIO Bar.
Memory32BitRegion = 0,
/// IO port Bar.
IORegion = 0x01,
IoRegion = 0x01,
/// 64-bit MMIO Bar.
Memory64BitRegion = 0x04,
/// Fake type for the upper Bar of 64-bit MMIO Bar.
Expand Down Expand Up @@ -469,7 +469,7 @@ impl BarProgrammingParams {

let mut resources = DeviceResources::new();
match self.bar_type {
PciBarRegionType::IORegion => resources.append(Resource::PioAddressRange {
PciBarRegionType::IoRegion => resources.append(Resource::PioAddressRange {
base: base as u16,
size: self.len as u16,
}),
Expand Down Expand Up @@ -504,7 +504,7 @@ impl PciBarState {
fn mask(&self) -> u32 {
match self.type_ {
None => 0,
Some(PciBarRegionType::IORegion) => !(self.size - 1),
Some(PciBarRegionType::IoRegion) => !(self.size - 1),
Some(PciBarRegionType::Memory32BitRegion) => !(self.size - 1),
Some(PciBarRegionType::Memory64BitRegion) => {
if self.size == 0 {
Expand Down Expand Up @@ -548,7 +548,7 @@ pub struct Vp2pCap {
impl Vp2pCap {
fn new(clique_id: u8) -> Self {
Vp2pCap {
id: PciCapabilityID::VendorSpecific as u8,
id: PciCapabilityId::VendorSpecific as u8,
next: 0,
length: 8,
sig_1: 0x50,
Expand Down Expand Up @@ -583,8 +583,8 @@ impl PciCapability for Vp2pCap {

fn write_u8(&mut self, _offset: usize, _value: u8) {}

fn pci_capability_type(&self) -> PciCapabilityID {
PciCapabilityID::VendorSpecific
fn pci_capability_type(&self) -> PciCapabilityId {
PciCapabilityId::VendorSpecific
}
}

Expand Down Expand Up @@ -999,7 +999,7 @@ impl PciConfiguration {
.checked_add(config.size - 1)
.ok_or(Error::BarAddressInvalid(config.addr, config.size))?;
match config.bar_type {
PciBarRegionType::IORegion => {
PciBarRegionType::IoRegion => {
if config.size < 0x4 || config.size > u64::from(u32::max_value()) {
return Err(Error::BarSizeInvalid(config.size));
}
Expand Down Expand Up @@ -1045,7 +1045,7 @@ impl PciConfiguration {
BAR_MEM_ADDR_MASK,
config.prefetchable as u32 | config.bar_type as u32,
),
PciBarRegionType::IORegion => (BAR_IO_ADDR_MASK, config.bar_type as u32),
PciBarRegionType::IoRegion => (BAR_IO_ADDR_MASK, config.bar_type as u32),
PciBarRegionType::Memory64BitRegionUpper => {
panic!("Invalid PCI Bar type");
}
Expand Down Expand Up @@ -1167,7 +1167,7 @@ impl PciConfiguration {
let value = self.registers[reg_idx] & mask;
if let Some(bar_type) = self.bar_type(bar_idx) {
match bar_type {
PciBarRegionType::Memory32BitRegion | PciBarRegionType::IORegion => {
PciBarRegionType::Memory32BitRegion | PciBarRegionType::IoRegion => {
if (value & mask) != self.bar_addr(bar_idx) {
debug!(
"DETECT BAR REPROG: current 0x{:x}, new 0x{:x}",
Expand Down Expand Up @@ -1264,7 +1264,7 @@ impl PciConfiguration {

assert!(!self.bar_allocated(param.bar_idx));
let constraint = match param.bar_type {
PciBarRegionType::IORegion => {
PciBarRegionType::IoRegion => {
let range = (
param.new_base as u16,
(param.new_base + param.len - 1) as u16,
Expand Down Expand Up @@ -1299,7 +1299,7 @@ impl PciConfiguration {
size: param.len,
}
}
PciBarRegionType::IORegion => Resource::PioAddressRange {
PciBarRegionType::IoRegion => Resource::PioAddressRange {
base: param.old_base as u16,
size: param.len as u16,
},
Expand Down Expand Up @@ -1448,8 +1448,8 @@ pub(crate) mod tests {
}
}

fn pci_capability_type(&self) -> PciCapabilityID {
PciCapabilityID::Test
fn pci_capability_type(&self) -> PciCapabilityId {
PciCapabilityId::Test
}
}

Expand All @@ -1460,7 +1460,7 @@ pub(crate) mod tests {

// Add two capabilities with different contents.
let cap1 = Box::new(TestCap {
id: PciCapabilityID::VendorSpecific as u8,
id: PciCapabilityId::VendorSpecific as u8,
next: 0,
len: 4,
foo: 0xAA,
Expand All @@ -1472,7 +1472,7 @@ pub(crate) mod tests {
assert_eq!(cap1_offset % 4, 0);

let cap2 = Box::new(TestCap {
id: PciCapabilityID::VendorSpecific as u8,
id: PciCapabilityId::VendorSpecific as u8,
next: 0,
len: 8,
foo: 0xBB,
Expand Down Expand Up @@ -1556,8 +1556,8 @@ pub(crate) mod tests {
assert_eq!(config.size(), 0);
assert_eq!(config.addr, 0);

config = config.set_bar_type(PciBarRegionType::IORegion);
assert_eq!(config.bar_type(), PciBarRegionType::IORegion);
config = config.set_bar_type(PciBarRegionType::IoRegion);
assert_eq!(config.bar_type(), PciBarRegionType::IoRegion);
config = config.set_address(0x1000);
assert_eq!(config.address(), 0x1000);
config = config.set_size(0x2000);
Expand Down Expand Up @@ -1744,7 +1744,7 @@ pub(crate) mod tests {
// Allocate BAR5
let bar = PciBarConfiguration {
bar_idx: 5,
bar_type: PciBarRegionType::IORegion,
bar_type: PciBarRegionType::IoRegion,
prefetchable: PciBarPrefetchable::NotPrefetchable,
addr: 0x2000,
size: 0x1000,
Expand Down
7 changes: 6 additions & 1 deletion src/dragonball/src/dbs_pci/src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ pub use bus::PciBus;
mod configuration;
pub use configuration::{
BarProgrammingParams, PciBarConfiguration, PciBarPrefetchable, PciBarRegionType,
PciBridgeSubclass, PciCapability, PciCapabilityID, PciClassCode, PciConfiguration,
PciBridgeSubclass, PciCapability, PciCapabilityId, PciClassCode, PciConfiguration,
PciHeaderType, PciInterruptPin, PciMassStorageSubclass, PciMultimediaSubclass,
PciNetworkControllerSubclass, PciProgrammingInterface, PciSerialBusSubClass, PciSubclass,
NUM_BAR_REGS, NUM_CONFIGURATION_REGISTERS,
Expand All @@ -48,7 +48,12 @@ mod root_device;
pub use root_device::PciRootDevice;

mod msi;
pub use msi::{MsiCap, MsiState};

mod msix;
pub use msix::{MsixCap, MsixState, MSIX_TABLE_ENTRY_SIZE};

mod vfio;

/// Error codes related to PCI root/bus/device operations.
#[derive(Debug, thiserror::Error)]
Expand Down
8 changes: 4 additions & 4 deletions src/dragonball/src/dbs_pci/src/msi.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ use dbs_interrupt::{
};
use log::{debug, error, warn};

use crate::configuration::{PciCapability, PciCapabilityID};
use crate::configuration::{PciCapability, PciCapabilityId};
use crate::fill_config_data;

// MSI control masks
Expand Down Expand Up @@ -114,7 +114,7 @@ pub struct MsiCap {
impl MsiCap {
/// Create a new PCI MSI capability structure.
pub fn new(next: u8, mut msg_ctl: u16) -> Self {
let cap_id_next = (next as u16) << 8 | PciCapabilityID::MessageSignalledInterrupts as u16;
let cap_id_next = (next as u16) << 8 | PciCapabilityId::MessageSignalledInterrupts as u16;

// By default MSI capability is disabled, and driver needs to explicitly turn it on.
msg_ctl &= !MSI_CTL_ENABLE;
Expand Down Expand Up @@ -457,8 +457,8 @@ impl PciCapability for MsiCap {
}
}

fn pci_capability_type(&self) -> PciCapabilityID {
PciCapabilityID::MessageSignalledInterrupts
fn pci_capability_type(&self) -> PciCapabilityId {
PciCapabilityId::MessageSignalledInterrupts
}
}

Expand Down
10 changes: 5 additions & 5 deletions src/dragonball/src/dbs_pci/src/msix.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ use dbs_interrupt::{
use log::debug;
use vm_memory::ByteValued;

use crate::configuration::{PciCapability, PciCapabilityID};
use crate::configuration::{PciCapability, PciCapabilityId};

const MAX_MSIX_VECTORS_PER_DEVICE: u16 = 2048;
const FUNCTION_MASK_BIT: u8 = 14;
Expand Down Expand Up @@ -72,7 +72,7 @@ impl MsixCap {
let msg_ctl: u16 = table_size - 1;

MsixCap {
cap_id: PciCapabilityID::MSIX as u8,
cap_id: PciCapabilityId::MSIX as u8,
cap_next: 0,
msg_ctl,
table: (table_off & MSIX_TABLE_OFFSET_MASK)
Expand Down Expand Up @@ -140,8 +140,8 @@ impl PciCapability for MsixCap {
}
}

fn pci_capability_type(&self) -> PciCapabilityID {
PciCapabilityID::MSIX
fn pci_capability_type(&self) -> PciCapabilityId {
PciCapabilityId::MSIX
}
}

Expand Down Expand Up @@ -495,7 +495,7 @@ mod tests {
assert_eq!(msix.table_size(), 0x100);

assert_eq!(msix.cap_next, 0);
assert_eq!(msix.cap_id, PciCapabilityID::MSIX as u8);
assert_eq!(msix.cap_id, PciCapabilityId::MSIX as u8);
let msg_ctl = msix.msg_ctl;
assert_eq!(msix.read_u16(0x2), msg_ctl);
msix.write_u16(0x2, MSIX_ENABLE_MASK);
Expand Down

0 comments on commit 8895cb8

Please sign in to comment.