Skip to content

Commit

Permalink
Merge tag 'pull-vfio-20231009' of https://github.com/legoater/qemu in…
Browse files Browse the repository at this point in the history 
…to staging

vfio queue:

* Fix for VFIO display when using Intel vGPUs
* Support for dynamic MSI-X

# -----BEGIN PGP SIGNATURE-----
#
# iQIzBAABCAAdFiEEoPZlSPBIlev+awtgUaNDx8/77KEFAmUjoLIACgkQUaNDx8/7
# 7KE+gw/9FTQFRkmlkSMlqRGjINF/VmfX6TsX+dy3ZB+aJia6qahco+u9hd3yQxiA
# /KI4FZnQCH/ZFizjR7hJdsxLnd+l989RFmoy+NTEXfgBMSLu4aU1UlVC1pyuhJ5L
# xadGQ2UIclD1Gz70laa9ketebLHdyc/Pku2xt9oreR6kRRFHZ3V4QhMNhcwGapO1
# 0wytLFXPVyGa7YYTB5qQPHPWyY9sM0n6E4E7jVnhfOw75cUVNvSr+9HlJbR1FN3Z
# 4klNMXayKGAZmh9oKpQWBsf4aUwLDu//eCk64TkQHp0pNrvRAJJBwgkhsI1FigeW
# SJ2JjQsIg/vLu2oyUhp2PJ59cQSMFZPgEqRhhRQ2RKhIfwOZY4kgfvKFtSHvWijV
# u0r8/HMIJE0fNffigyDlfLCsUEYu3OuJXMlU+5xrwi77hWlPrGb8D1J7LhwUnldk
# kZaw9VEranlbMQT773cMA7f/pgS1Sc6CkdqfJLGIHA4PsEk44Lzen2BzRroz8+Km
# tn8hHt+GQK/ZGKmOPXWm44Bd48Be08cMz/pOI2cqoScEKKEQ8HUul3H1/k8sqauh
# 1gPo1hIPXo/GaGRvUvPsj4cK8oQm77EHksEQ4Nxvn+ZWTW2FnMQkb9QFbF8bTmEo
# KiJJ6s8qbd1CWGYbO0GSE8ss3NUZq1YbWsMXmUP0JccEgvjeL2M=
# =QRhQ
# -----END PGP SIGNATURE-----
# gpg: Signature made Mon 09 Oct 2023 02:41:54 EDT
# gpg:                using RSA key A0F66548F04895EBFE6B0B6051A343C7CFFBECA1
# gpg: Good signature from "Cédric Le Goater <clg@redhat.com>" [unknown]
# gpg:                 aka "Cédric Le Goater <clg@kaod.org>" [unknown]
# gpg: WARNING: This key is not certified with a trusted signature!
# gpg:          There is no indication that the signature belongs to the owner.
# Primary key fingerprint: A0F6 6548 F048 95EB FE6B  0B60 51A3 43C7 CFFB ECA1

* tag 'pull-vfio-20231009' of https://github.com/legoater/qemu:
  vfio/pci: enable MSI-X in interrupt restoring on dynamic allocation
  vfio/pci: use an invalid fd to enable MSI-X
  vfio/pci: enable vector on dynamic MSI-X allocation
  vfio/pci: detect the support of dynamic MSI-X allocation
  vfio/pci: rename vfio_put_device to vfio_pci_put_device
  vfio/display: Fix missing update to set backing fields

Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
  • Loading branch information
@stefanhaRH
stefanhaRH committed Oct 9, 2023
2 parents 1527c6b + eaadba6 commit cea3ea6
Show file tree
Hide file tree
Showing 4 changed files with 101 additions and 31 deletions.
2 changes: 2 additions & 0 deletions hw/vfio/display.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -243,6 +243,8 @@ static VFIODMABuf *vfio_display_get_dmabuf(VFIOPCIDevice *vdev,
dmabuf->dmabuf_id = plane.dmabuf_id;
dmabuf->buf.width = plane.width;
dmabuf->buf.height = plane.height;
dmabuf->buf.backing_width = plane.width;
dmabuf->buf.backing_height = plane.height;
dmabuf->buf.stride = plane.stride;
dmabuf->buf.fourcc = plane.drm_format;
dmabuf->buf.modifier = plane.drm_format_mod;
Expand Down
127 changes: 97 additions & 30 deletions hw/vfio/pci.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -369,12 +369,56 @@ static void vfio_msi_interrupt(void *opaque)
notify(&vdev->pdev, nr);
}

/*
* Get MSI-X enabled, but no vector enabled, by setting vector 0 with an invalid
* fd to kernel.
*/
static int vfio_enable_msix_no_vec(VFIOPCIDevice *vdev)
{
g_autofree struct vfio_irq_set *irq_set = NULL;
int ret = 0, argsz;
int32_t *fd;

argsz = sizeof(*irq_set) + sizeof(*fd);

irq_set = g_malloc0(argsz);
irq_set->argsz = argsz;
irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD |
VFIO_IRQ_SET_ACTION_TRIGGER;
irq_set->index = VFIO_PCI_MSIX_IRQ_INDEX;
irq_set->start = 0;
irq_set->count = 1;
fd = (int32_t *)&irq_set->data;
*fd = -1;

ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_SET_IRQS, irq_set);

return ret;
}

static int vfio_enable_vectors(VFIOPCIDevice *vdev, bool msix)
{
struct vfio_irq_set *irq_set;
int ret = 0, i, argsz;
int32_t *fds;

/*
* If dynamic MSI-X allocation is supported, the vectors to be allocated
* and enabled can be scattered. Before kernel enabling MSI-X, setting
* nr_vectors causes all these vectors to be allocated on host.
*
* To keep allocation as needed, use vector 0 with an invalid fd to get
* MSI-X enabled first, then set vectors with a potentially sparse set of
* eventfds to enable interrupts only when enabled in guest.
*/
if (msix && !vdev->msix->noresize) {
ret = vfio_enable_msix_no_vec(vdev);

if (ret) {
return ret;
}
}

argsz = sizeof(*irq_set) + (vdev->nr_vectors * sizeof(*fds));

irq_set = g_malloc0(argsz);
Expand Down Expand Up @@ -470,6 +514,7 @@ static int vfio_msix_vector_do_use(PCIDevice *pdev, unsigned int nr,
VFIOPCIDevice *vdev = VFIO_PCI(pdev);
VFIOMSIVector *vector;
int ret;
bool resizing = !!(vdev->nr_vectors < nr + 1);

trace_vfio_msix_vector_do_use(vdev->vbasedev.name, nr);

Expand Down Expand Up @@ -512,33 +557,42 @@ static int vfio_msix_vector_do_use(PCIDevice *pdev, unsigned int nr,
}

/*
* We don't want to have the host allocate all possible MSI vectors
* for a device if they're not in use, so we shutdown and incrementally
* increase them as needed.
* When dynamic allocation is not supported, we don't want to have the
* host allocate all possible MSI vectors for a device if they're not
* in use, so we shutdown and incrementally increase them as needed.
* nr_vectors represents the total number of vectors allocated.
*
* When dynamic allocation is supported, let the host only allocate
* and enable a vector when it is in use in guest. nr_vectors represents
* the upper bound of vectors being enabled (but not all of the ranges
* is allocated or enabled).
*/
if (vdev->nr_vectors < nr + 1) {
if (resizing) {
vdev->nr_vectors = nr + 1;
if (!vdev->defer_kvm_irq_routing) {
}

if (!vdev->defer_kvm_irq_routing) {
if (vdev->msix->noresize && resizing) {
vfio_disable_irqindex(&vdev->vbasedev, VFIO_PCI_MSIX_IRQ_INDEX);
ret = vfio_enable_vectors(vdev, true);
if (ret) {
error_report("vfio: failed to enable vectors, %d", ret);
}
}
} else {
Error *err = NULL;
int32_t fd;

if (vector->virq >= 0) {
fd = event_notifier_get_fd(&vector->kvm_interrupt);
} else {
fd = event_notifier_get_fd(&vector->interrupt);
}
Error *err = NULL;
int32_t fd;

if (vfio_set_irq_signaling(&vdev->vbasedev,
VFIO_PCI_MSIX_IRQ_INDEX, nr,
VFIO_IRQ_SET_ACTION_TRIGGER, fd, &err)) {
error_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name);
if (vector->virq >= 0) {
fd = event_notifier_get_fd(&vector->kvm_interrupt);
} else {
fd = event_notifier_get_fd(&vector->interrupt);
}

if (vfio_set_irq_signaling(&vdev->vbasedev,
VFIO_PCI_MSIX_IRQ_INDEX, nr,
VFIO_IRQ_SET_ACTION_TRIGGER, fd, &err)) {
error_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name);
}
}
}

Expand Down Expand Up @@ -608,6 +662,8 @@ static void vfio_commit_kvm_msi_virq_batch(VFIOPCIDevice *vdev)

static void vfio_msix_enable(VFIOPCIDevice *vdev)
{
int ret;

vfio_disable_interrupts(vdev);

vdev->msi_vectors = g_new0(VFIOMSIVector, vdev->msix->entries);
Expand All @@ -630,8 +686,6 @@ static void vfio_msix_enable(VFIOPCIDevice *vdev)
vfio_commit_kvm_msi_virq_batch(vdev);

if (vdev->nr_vectors) {
int ret;

ret = vfio_enable_vectors(vdev, true);
if (ret) {
error_report("vfio: failed to enable vectors, %d", ret);
Expand All @@ -645,13 +699,14 @@ static void vfio_msix_enable(VFIOPCIDevice *vdev)
* MSI-X capability, but leaves the vector table masked. We therefore
* can't rely on a vector_use callback (from request_irq() in the guest)
* to switch the physical device into MSI-X mode because that may come a
* long time after pci_enable_msix(). This code enables vector 0 with
* triggering to userspace, then immediately release the vector, leaving
* the physical device with no vectors enabled, but MSI-X enabled, just
* like the guest view.
* long time after pci_enable_msix(). This code sets vector 0 with an
* invalid fd to make the physical device MSI-X enabled, but with no
* vectors enabled, just like the guest view.
*/
vfio_msix_vector_do_use(&vdev->pdev, 0, NULL, NULL);
vfio_msix_vector_release(&vdev->pdev, 0);
ret = vfio_enable_msix_no_vec(vdev);
if (ret) {
error_report("vfio: failed to enable MSI-X, %d", ret);
}
}

trace_vfio_msix_enable(vdev->vbasedev.name);
Expand Down Expand Up @@ -1493,7 +1548,9 @@ static void vfio_msix_early_setup(VFIOPCIDevice *vdev, Error **errp)
uint8_t pos;
uint16_t ctrl;
uint32_t table, pba;
int fd = vdev->vbasedev.fd;
int ret, fd = vdev->vbasedev.fd;
struct vfio_irq_info irq_info = { .argsz = sizeof(irq_info),
.index = VFIO_PCI_MSIX_IRQ_INDEX };
VFIOMSIXInfo *msix;

pos = pci_find_capability(&vdev->pdev, PCI_CAP_ID_MSIX);
Expand Down Expand Up @@ -1530,6 +1587,15 @@ static void vfio_msix_early_setup(VFIOPCIDevice *vdev, Error **errp)
msix->pba_offset = pba & ~PCI_MSIX_FLAGS_BIRMASK;
msix->entries = (ctrl & PCI_MSIX_FLAGS_QSIZE) + 1;

ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_GET_IRQ_INFO, &irq_info);
if (ret < 0) {
error_setg_errno(errp, -ret, "failed to get MSI-X irq info");
g_free(msix);
return;
}

msix->noresize = !!(irq_info.flags & VFIO_IRQ_INFO_NORESIZE);

/*
* Test the size of the pba_offset variable and catch if it extends outside
* of the specified BAR. If it is the case, we need to apply a hardware
Expand Down Expand Up @@ -1562,7 +1628,8 @@ static void vfio_msix_early_setup(VFIOPCIDevice *vdev, Error **errp)
}

trace_vfio_msix_early_setup(vdev->vbasedev.name, pos, msix->table_bar,
msix->table_offset, msix->entries);
msix->table_offset, msix->entries,
msix->noresize);
vdev->msix = msix;

vfio_pci_fixup_msix_region(vdev);
Expand Down Expand Up @@ -2826,7 +2893,7 @@ static void vfio_populate_device(VFIOPCIDevice *vdev, Error **errp)
}
}

static void vfio_put_device(VFIOPCIDevice *vdev)
static void vfio_pci_put_device(VFIOPCIDevice *vdev)
{
g_free(vdev->vbasedev.name);
g_free(vdev->msix);
Expand Down Expand Up @@ -3317,7 +3384,7 @@ static void vfio_instance_finalize(Object *obj)
*
* g_free(vdev->igd_opregion);
*/
vfio_put_device(vdev);
vfio_pci_put_device(vdev);
vfio_put_group(group);
}

Expand Down
1 change: 1 addition & 0 deletions hw/vfio/pci.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,7 @@ typedef struct VFIOMSIXInfo {
uint32_t table_offset;
uint32_t pba_offset;
unsigned long *pending;
bool noresize;
} VFIOMSIXInfo;

#define TYPE_VFIO_PCI "vfio-pci"
Expand Down
2 changes: 1 addition & 1 deletion hw/vfio/trace-events
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ vfio_vga_read(uint64_t addr, int size, uint64_t data) " (0x%"PRIx64", %d) = 0x%"
vfio_pci_read_config(const char *name, int addr, int len, int val) " (%s, @0x%x, len=0x%x) 0x%x"
vfio_pci_write_config(const char *name, int addr, int val, int len) " (%s, @0x%x, 0x%x, len=0x%x)"
vfio_msi_setup(const char *name, int pos) "%s PCI MSI CAP @0x%x"
vfio_msix_early_setup(const char *name, int pos, int table_bar, int offset, int entries) "%s PCI MSI-X CAP @0x%x, BAR %d, offset 0x%x, entries %d"
vfio_msix_early_setup(const char *name, int pos, int table_bar, int offset, int entries, bool noresize) "%s PCI MSI-X CAP @0x%x, BAR %d, offset 0x%x, entries %d, noresize %d"
vfio_check_pcie_flr(const char *name) "%s Supports FLR via PCIe cap"
vfio_check_pm_reset(const char *name) "%s Supports PM reset"
vfio_check_af_flr(const char *name) "%s Supports FLR via AF cap"
Expand Down

0 comments on commit cea3ea6

Please sign in to comment.