Skip to content

Commit

Permalink
Merge pull request #9551 from pmores/support-iommu
Browse files Browse the repository at this point in the history 
runtime-rs: support IOMMU in qemu VMs
  • Loading branch information
@pmores
pmores committed Apr 29, 2024
2 parents 7b039eb + 908ec31 commit 1dd06cf
Showing 1 changed file with 102 additions and 5 deletions.
107 changes: 102 additions & 5 deletions src/runtime-rs/crates/hypervisor/src/qemu/cmdline_generator.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ trait ToQemuParams: Send + Sync {
async fn qemu_params(&self) -> Result<Vec<String>>;
}

#[derive(Debug)]
#[derive(Debug, PartialEq)]
enum VirtioBusType {
Pci,
Ccw,
Expand Down Expand Up @@ -282,6 +282,7 @@ struct Machine {
accel: String,
options: String,
nvdimm: bool,
kernel_irqchip: Option<String>,

is_nvdimm_supported: bool,
memory_backend: Option<String>,
Expand Down Expand Up @@ -309,6 +310,7 @@ impl Machine {
accel: "kvm".to_owned(),
options: config.machine_info.machine_accelerators.clone(),
nvdimm: false,
kernel_irqchip: None,
is_nvdimm_supported,
memory_backend: None,
}
Expand All @@ -326,6 +328,11 @@ impl Machine {
self.memory_backend = Some(mem_backend.to_owned());
self
}

fn set_kernel_irqchip(&mut self, kernel_irqchip: &str) -> &mut Self {
self.kernel_irqchip = Some(kernel_irqchip.to_owned());
self
}
}

#[async_trait]
Expand All @@ -340,6 +347,9 @@ impl ToQemuParams for Machine {
if self.nvdimm {
params.push("nvdimm=on".to_owned());
}
if let Some(kernel_irqchip) = &self.kernel_irqchip {
params.push(format!("kernel_irqchip={}", kernel_irqchip));
}
if let Some(mem_backend) = &self.memory_backend {
params.push(format!("memory-backend={}", mem_backend));
}
Expand Down Expand Up @@ -813,6 +823,7 @@ struct VhostVsock {
vhostfd: tokio::fs::File,
guest_cid: u32,
disable_modern: bool,
iommu_platform: bool,
}

impl VhostVsock {
Expand All @@ -822,13 +833,19 @@ impl VhostVsock {
vhostfd,
guest_cid,
disable_modern: false,
iommu_platform: false,
}
}

fn set_disable_modern(&mut self, disable_modern: bool) -> &mut Self {
self.disable_modern = disable_modern;
self
}

fn set_iommu_platform(&mut self, iommu_platform: bool) -> &mut Self {
self.iommu_platform = iommu_platform;
self
}
}

#[async_trait]
Expand All @@ -839,6 +856,9 @@ impl ToQemuParams for VhostVsock {
if self.disable_modern {
params.push("disable-modern=true".to_owned());
}
if self.iommu_platform {
params.push("iommu_platform=on".to_owned());
}
params.push(format!("vhostfd={}", self.vhostfd.as_raw_fd()));
params.push(format!("guest-cid={}", self.guest_cid));

Expand Down Expand Up @@ -977,6 +997,7 @@ pub struct DeviceVirtioNet {
disable_modern: bool,

num_queues: u32,
iommu_platform: bool,
}

impl DeviceVirtioNet {
Expand All @@ -987,6 +1008,7 @@ impl DeviceVirtioNet {
mac_address,
disable_modern: false,
num_queues: 1,
iommu_platform: false,
}
}

Expand All @@ -999,6 +1021,11 @@ impl DeviceVirtioNet {
self.num_queues = num_queues;
self
}

fn set_iommu_platform(&mut self, iommu_platform: bool) -> &mut Self {
self.iommu_platform = iommu_platform;
self
}
}

#[async_trait]
Expand All @@ -1015,6 +1042,9 @@ impl ToQemuParams for DeviceVirtioNet {
if self.disable_modern {
params.push("disable-modern=true".to_owned());
}
if self.iommu_platform {
params.push("iommu_platform=on".to_owned());
}

params.push("mq=on".to_owned());
params.push(format!("vectors={}", 2 * self.num_queues + 2));
Expand All @@ -1027,15 +1057,22 @@ impl ToQemuParams for DeviceVirtioNet {
struct DeviceVirtioSerial {
id: String,
bus_type: VirtioBusType,
iommu_platform: bool,
}

impl DeviceVirtioSerial {
fn new(id: &str, bus_type: VirtioBusType) -> DeviceVirtioSerial {
DeviceVirtioSerial {
id: id.to_owned(),
bus_type,
iommu_platform: false,
}
}

fn set_iommu_platform(&mut self, iommu_platform: bool) -> &mut Self {
self.iommu_platform = iommu_platform;
self
}
}

#[async_trait]
Expand All @@ -1044,6 +1081,9 @@ impl ToQemuParams for DeviceVirtioSerial {
let mut params = Vec::new();
params.push(format!("virtio-serial-{}", self.bus_type));
params.push(format!("id={}", self.id));
if self.iommu_platform {
params.push("iommu_platform=on".to_owned());
}
Ok(vec!["-device".to_owned(), params.join(",")])
}
}
Expand Down Expand Up @@ -1108,6 +1148,36 @@ impl ToQemuParams for Rtc {
}
}

#[derive(Debug)]
struct DeviceIntelIommu {
intremap: bool,
device_iotlb: bool,
caching_mode: bool,
}

impl DeviceIntelIommu {
fn new() -> DeviceIntelIommu {
DeviceIntelIommu {
intremap: true,
device_iotlb: true,
caching_mode: true,
}
}
}

#[async_trait]
impl ToQemuParams for DeviceIntelIommu {
async fn qemu_params(&self) -> Result<Vec<String>> {
let mut params = Vec::new();
params.push("intel-iommu".to_owned());
let to_onoff = |b| if b { "on" } else { "off" };
params.push(format!("intremap={}", to_onoff(self.intremap)));
params.push(format!("device-iotlb={}", to_onoff(self.device_iotlb)));
params.push(format!("caching-mode={}", to_onoff(self.caching_mode)));
Ok(vec!["-device".to_owned(), params.join(",")])
}
}

fn is_running_in_vm() -> Result<bool> {
let res = read_to_string("/proc/cpuinfo")?
.lines()
Expand Down Expand Up @@ -1151,7 +1221,7 @@ pub struct QemuCmdLine<'a> {

impl<'a> QemuCmdLine<'a> {
pub fn new(id: &str, config: &'a HypervisorConfig) -> Result<QemuCmdLine<'a>> {
Ok(QemuCmdLine {
let mut qemu_cmd_line = QemuCmdLine {
id: id.to_string(),
config,
kernel: Kernel::new(config)?,
Expand All @@ -1162,7 +1232,13 @@ impl<'a> QemuCmdLine<'a> {
rtc: Rtc::new(),
knobs: Knobs::new(config),
devices: Vec::new(),
})
};

if config.device_info.enable_iommu {
qemu_cmd_line.add_iommu();
}

Ok(qemu_cmd_line)
}

fn bus_type(&self) -> VirtioBusType {
Expand All @@ -1173,6 +1249,17 @@ impl<'a> QemuCmdLine<'a> {
}
}

fn add_iommu(&mut self) {
let dev_iommu = DeviceIntelIommu::new();
self.devices.push(Box::new(dev_iommu));

self.kernel
.params
.append(&mut KernelParams::from_string("intel_iommu=on iommu=pt"));

self.machine.set_kernel_irqchip("split");
}

pub fn add_virtiofs_share(
&mut self,
virtiofsd_socket_path: &str,
Expand All @@ -1191,7 +1278,7 @@ impl<'a> QemuCmdLine<'a> {

let mut virtiofs_device = DeviceVhostUserFs::new(chardev_name, mount_tag, self.bus_type());
virtiofs_device.set_queue_size(queue_size);
if self.config.device_info.enable_iommu_platform {
if self.config.device_info.enable_iommu_platform && self.bus_type() == VirtioBusType::Ccw {
virtiofs_device.set_iommu_platform(true);
}
self.devices.push(Box::new(virtiofs_device));
Expand Down Expand Up @@ -1225,6 +1312,10 @@ impl<'a> QemuCmdLine<'a> {
vhost_vsock_pci.set_disable_modern(true);
}

if self.config.device_info.enable_iommu_platform {
vhost_vsock_pci.set_iommu_platform(true);
}

self.devices.push(Box::new(vhost_vsock_pci));
Ok(())
}
Expand Down Expand Up @@ -1297,6 +1388,9 @@ impl<'a> QemuCmdLine<'a> {
if should_disable_modern() {
virtio_net_device.set_disable_modern(true);
}
if self.config.device_info.enable_iommu_platform && self.bus_type() == VirtioBusType::Ccw {
virtio_net_device.set_iommu_platform(true);
}
if self.config.network_info.network_queues > 1 {
virtio_net_device.set_num_queues(self.config.network_info.network_queues);
}
Expand All @@ -1307,7 +1401,10 @@ impl<'a> QemuCmdLine<'a> {
}

pub fn add_console(&mut self, console_socket_path: &str) {
let serial_dev = DeviceVirtioSerial::new("serial0", self.bus_type());
let mut serial_dev = DeviceVirtioSerial::new("serial0", self.bus_type());
if self.config.device_info.enable_iommu_platform && self.bus_type() == VirtioBusType::Ccw {
serial_dev.set_iommu_platform(true);
}
self.devices.push(Box::new(serial_dev));

let chardev_name = "charconsole0";
Expand Down

0 comments on commit 1dd06cf

Please sign in to comment.