Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
Merge tag 'pull-vfio-20230710' of https://github.com/legoater/qemu in… 
…to staging

vfio queue:

* Fixes in error handling paths of VFIO PCI devices
* Improvements of reported errors for VFIO migration
* Linux header update
* Enablement of AtomicOps completers on root ports
* Fix for unplug of passthrough AP devices

# -----BEGIN PGP SIGNATURE-----
#
# iQIzBAABCAAdFiEEoPZlSPBIlev+awtgUaNDx8/77KEFAmSrug0ACgkQUaNDx8/7
# 7KHYCRAAt6UeZi8nKPlN+cs6guOagCcAJOu13nm7XN0bFxjYf/Q2t618cpM7PLSk
# h+4VGsMUVJ1dumcCkBmv7LAn0G6CpVR3VDi5QuGfMODRhpWfSoaypPIizRgrbarL
# lSyaVaPIaddlDZ4AIfFA9Ebnytvm5/ecsyTr0cv7OejVKWI/jN6bC/v36AmNQKKQ
# J5RCDpQ6fOsdqf0Dzvn7xjuHRE4DYtsWkVoslDoBQMgPWHLF8UwRu/OPD6cBQYAR
# /fmgoOkkNDMdN3laqwAyfAUjKfOFpLuZzJ5KNFjtkBiktm66dw4Y8/lWoChVR+S6
# PRZ3nk0HxyzB96zCytfggBX905PBD54LIuockRaYKTlTxT19C3fDjDz5tsjKNhLR
# aFec4KiJaUJj0fa/Vw8DB/WUbCgbOXGHiWhY8vNdpVoc9AZe8xj9z4nB3hmzx1i/
# lZhsM/s3kTNHpVGlW7vTfbToFBmt1eoglu+ILe/HeHLi8LjzCsHy+wR5c0n0/HVI
# fLUuUS1AGQvi8+HCCUi7gwzpJkl4rPJsPx51wfXJk+q/3GQ8g9Mg9qotHNHm4N60
# zq/I5VqqEkJzdaMjup04ZqsMAWqGrnU2f4aNPvBhgaeO9CQE/buIsA34buQRwiG4
# wTodqm0jrkx0Z59jliZ0mFU/LxMvhMaQCEh+OdyZ9vRtfLBjF4c=
# =U2Hc
# -----END PGP SIGNATURE-----
# gpg: Signature made Mon 10 Jul 2023 08:58:05 AM BST
# gpg:                using RSA key A0F66548F04895EBFE6B0B6051A343C7CFFBECA1
# gpg: Good signature from "Cédric Le Goater <clg@kaod.org>" [undefined]
# gpg: WARNING: This key is not certified with a trusted signature!
# gpg:          There is no indication that the signature belongs to the owner.
# Primary key fingerprint: A0F6 6548 F048 95EB FE6B  0B60 51A3 43C7 CFFB ECA1

* tag 'pull-vfio-20230710' of https://github.com/legoater/qemu:
  vfio/pci: Enable AtomicOps completers on root ports
  pcie: Add a PCIe capability version helper
  s390x/ap: Wire up the device request notifier interface
  linux-headers: update to v6.5-rc1
  vfio: Fix null pointer dereference bug in vfio_bars_finalize()
  vfio/migration: Return bool type for vfio_migration_realize()
  vfio/migration: Remove print of "Migration disabled"
  vfio/migration: Free resources when vfio_migration_realize fails
  vfio/migration: Change vIOMMU blocker from global to per device
  vfio/pci: Disable INTx in vfio_realize error path
  hw/vfio/pci-quirks: Sanitize capability pointer

Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
  • Loading branch information
@rth7680
rth7680 committed Jul 10, 2023
2 parents 2ff49e9 + c00aac6 commit fcb237e
Show file tree
Hide file tree
Showing 41 changed files with 678 additions and 229 deletions.
7 changes: 7 additions & 0 deletions hw/pci/pcie.c
View file
Expand Up @@ -274,6 +274,13 @@ uint8_t pcie_cap_get_type(const PCIDevice *dev)
PCI_EXP_FLAGS_TYPE) >> PCI_EXP_FLAGS_TYPE_SHIFT;
}

uint8_t pcie_cap_get_version(const PCIDevice *dev)
{
uint32_t pos = dev->exp.exp_cap;
assert(pos > 0);
return pci_get_word(dev->config + pos + PCI_EXP_FLAGS) & PCI_EXP_FLAGS_VERS;
}

/* MSI/MSI-X */
/* pci express interrupt message number */
/* 7.8.2 PCI Express Capabilities Register: Interrupt Message Number */
Expand Down
113 changes: 113 additions & 0 deletions hw/vfio/ap.c
View file
Expand Up @@ -18,6 +18,8 @@
#include "hw/vfio/vfio-common.h"
#include "hw/s390x/ap-device.h"
#include "qemu/error-report.h"
#include "qemu/event_notifier.h"
#include "qemu/main-loop.h"
#include "qemu/module.h"
#include "qemu/option.h"
#include "qemu/config-file.h"
Expand All @@ -33,6 +35,7 @@
struct VFIOAPDevice {
APDevice apdev;
VFIODevice vdev;
EventNotifier req_notifier;
};

OBJECT_DECLARE_SIMPLE_TYPE(VFIOAPDevice, VFIO_AP_DEVICE)
Expand Down Expand Up @@ -84,10 +87,110 @@ static VFIOGroup *vfio_ap_get_group(VFIOAPDevice *vapdev, Error **errp)
return vfio_get_group(groupid, &address_space_memory, errp);
}

static void vfio_ap_req_notifier_handler(void *opaque)
{
VFIOAPDevice *vapdev = opaque;
Error *err = NULL;

if (!event_notifier_test_and_clear(&vapdev->req_notifier)) {
return;
}

qdev_unplug(DEVICE(vapdev), &err);

if (err) {
warn_reportf_err(err, VFIO_MSG_PREFIX, vapdev->vdev.name);
}
}

static void vfio_ap_register_irq_notifier(VFIOAPDevice *vapdev,
unsigned int irq, Error **errp)
{
int fd;
size_t argsz;
IOHandler *fd_read;
EventNotifier *notifier;
struct vfio_irq_info *irq_info;
VFIODevice *vdev = &vapdev->vdev;

switch (irq) {
case VFIO_AP_REQ_IRQ_INDEX:
notifier = &vapdev->req_notifier;
fd_read = vfio_ap_req_notifier_handler;
break;
default:
error_setg(errp, "vfio: Unsupported device irq(%d)", irq);
return;
}

if (vdev->num_irqs < irq + 1) {
error_setg(errp, "vfio: IRQ %u not available (number of irqs %u)",
irq, vdev->num_irqs);
return;
}

argsz = sizeof(*irq_info);
irq_info = g_malloc0(argsz);
irq_info->index = irq;
irq_info->argsz = argsz;

if (ioctl(vdev->fd, VFIO_DEVICE_GET_IRQ_INFO,
irq_info) < 0 || irq_info->count < 1) {
error_setg_errno(errp, errno, "vfio: Error getting irq info");
goto out_free_info;
}

if (event_notifier_init(notifier, 0)) {
error_setg_errno(errp, errno,
"vfio: Unable to init event notifier for irq (%d)",
irq);
goto out_free_info;
}

fd = event_notifier_get_fd(notifier);
qemu_set_fd_handler(fd, fd_read, NULL, vapdev);

if (vfio_set_irq_signaling(vdev, irq, 0, VFIO_IRQ_SET_ACTION_TRIGGER, fd,
errp)) {
qemu_set_fd_handler(fd, NULL, NULL, vapdev);
event_notifier_cleanup(notifier);
}

out_free_info:
g_free(irq_info);

}

static void vfio_ap_unregister_irq_notifier(VFIOAPDevice *vapdev,
unsigned int irq)
{
Error *err = NULL;
EventNotifier *notifier;

switch (irq) {
case VFIO_AP_REQ_IRQ_INDEX:
notifier = &vapdev->req_notifier;
break;
default:
error_report("vfio: Unsupported device irq(%d)", irq);
return;
}

if (vfio_set_irq_signaling(&vapdev->vdev, irq, 0,
VFIO_IRQ_SET_ACTION_TRIGGER, -1, &err)) {
warn_reportf_err(err, VFIO_MSG_PREFIX, vapdev->vdev.name);
}

qemu_set_fd_handler(event_notifier_get_fd(notifier),
NULL, NULL, vapdev);
event_notifier_cleanup(notifier);
}

static void vfio_ap_realize(DeviceState *dev, Error **errp)
{
int ret;
char *mdevid;
Error *err = NULL;
VFIOGroup *vfio_group;
APDevice *apdev = AP_DEVICE(dev);
VFIOAPDevice *vapdev = VFIO_AP_DEVICE(apdev);
Expand Down Expand Up @@ -116,6 +219,15 @@ static void vfio_ap_realize(DeviceState *dev, Error **errp)
goto out_get_dev_err;
}

vfio_ap_register_irq_notifier(vapdev, VFIO_AP_REQ_IRQ_INDEX, &err);
if (err) {
/*
* Report this error, but do not make it a failing condition.
* Lack of this IRQ in the host does not prevent normal operation.
*/
error_report_err(err);
}

return;

out_get_dev_err:
Expand All @@ -129,6 +241,7 @@ static void vfio_ap_unrealize(DeviceState *dev)
VFIOAPDevice *vapdev = VFIO_AP_DEVICE(apdev);
VFIOGroup *group = vapdev->vdev.group;

vfio_ap_unregister_irq_notifier(vapdev, VFIO_AP_REQ_IRQ_INDEX);
vfio_ap_put_device(vapdev);
vfio_put_group(group);
}
Expand Down
51 changes: 2 additions & 49 deletions hw/vfio/common.c
View file
Expand Up @@ -362,7 +362,6 @@ bool vfio_mig_active(void)
}

static Error *multiple_devices_migration_blocker;
static Error *giommu_migration_blocker;

static unsigned int vfio_migratable_device_num(void)
{
Expand Down Expand Up @@ -420,55 +419,9 @@ void vfio_unblock_multiple_devices_migration(void)
multiple_devices_migration_blocker = NULL;
}

static bool vfio_viommu_preset(void)
bool vfio_viommu_preset(VFIODevice *vbasedev)
{
VFIOAddressSpace *space;

QLIST_FOREACH(space, &vfio_address_spaces, list) {
if (space->as != &address_space_memory) {
return true;
}
}

return false;
}

int vfio_block_giommu_migration(VFIODevice *vbasedev, Error **errp)
{
int ret;

if (giommu_migration_blocker ||
!vfio_viommu_preset()) {
return 0;
}

if (vbasedev->enable_migration == ON_OFF_AUTO_ON) {
error_setg(errp,
"Migration is currently not supported with vIOMMU enabled");
return -EINVAL;
}

error_setg(&giommu_migration_blocker,
"Migration is currently not supported with vIOMMU enabled");
ret = migrate_add_blocker(giommu_migration_blocker, errp);
if (ret < 0) {
error_free(giommu_migration_blocker);
giommu_migration_blocker = NULL;
}

return ret;
}

void vfio_migration_finalize(void)
{
if (!giommu_migration_blocker ||
vfio_viommu_preset()) {
return;
}

migrate_del_blocker(giommu_migration_blocker);
error_free(giommu_migration_blocker);
giommu_migration_blocker = NULL;
return vbasedev->group->container->space->as != &address_space_memory;
}

static void vfio_set_migration_error(int err)
Expand Down
51 changes: 35 additions & 16 deletions hw/vfio/migration.c
View file
Expand Up @@ -802,6 +802,17 @@ static int vfio_migration_init(VFIODevice *vbasedev)
return 0;
}

static void vfio_migration_deinit(VFIODevice *vbasedev)
{
VFIOMigration *migration = vbasedev->migration;

remove_migration_state_change_notifier(&migration->migration_state);
qemu_del_vm_change_state_handler(migration->vm_state);
unregister_savevm(VMSTATE_IF(vbasedev->dev), "vfio", vbasedev);
vfio_migration_free(vbasedev);
vfio_unblock_multiple_devices_migration();
}

static int vfio_block_migration(VFIODevice *vbasedev, Error *err, Error **errp)
{
int ret;
Expand Down Expand Up @@ -835,15 +846,20 @@ void vfio_reset_bytes_transferred(void)
bytes_transferred = 0;
}

int vfio_migration_realize(VFIODevice *vbasedev, Error **errp)
/*
* Return true when either migration initialized or blocker registered.
* Currently only return false when adding blocker fails which will
* de-register vfio device.
*/
bool vfio_migration_realize(VFIODevice *vbasedev, Error **errp)
{
Error *err = NULL;
int ret;

if (vbasedev->enable_migration == ON_OFF_AUTO_OFF) {
error_setg(&err, "%s: Migration is disabled for VFIO device",
vbasedev->name);
return vfio_block_migration(vbasedev, err, errp);
return !vfio_block_migration(vbasedev, err, errp);
}

ret = vfio_migration_init(vbasedev);
Expand All @@ -858,15 +874,15 @@ int vfio_migration_realize(VFIODevice *vbasedev, Error **errp)
vbasedev->name, ret, strerror(-ret));
}

return vfio_block_migration(vbasedev, err, errp);
return !vfio_block_migration(vbasedev, err, errp);
}

if (!vbasedev->dirty_pages_supported) {
if (vbasedev->enable_migration == ON_OFF_AUTO_AUTO) {
error_setg(&err,
"%s: VFIO device doesn't support device dirty tracking",
vbasedev->name);
return vfio_block_migration(vbasedev, err, errp);
goto add_blocker;
}

warn_report("%s: VFIO device doesn't support device dirty tracking",
Expand All @@ -875,28 +891,31 @@ int vfio_migration_realize(VFIODevice *vbasedev, Error **errp)

ret = vfio_block_multiple_devices_migration(vbasedev, errp);
if (ret) {
return ret;
goto out_deinit;
}

ret = vfio_block_giommu_migration(vbasedev, errp);
if (ret) {
return ret;
if (vfio_viommu_preset(vbasedev)) {
error_setg(&err, "%s: Migration is currently not supported "
"with vIOMMU enabled", vbasedev->name);
goto add_blocker;
}

trace_vfio_migration_realize(vbasedev->name);
return 0;
return true;

add_blocker:
ret = vfio_block_migration(vbasedev, err, errp);
out_deinit:
if (ret) {
vfio_migration_deinit(vbasedev);
}
return !ret;
}

void vfio_migration_exit(VFIODevice *vbasedev)
{
if (vbasedev->migration) {
VFIOMigration *migration = vbasedev->migration;

remove_migration_state_change_notifier(&migration->migration_state);
qemu_del_vm_change_state_handler(migration->vm_state);
unregister_savevm(VMSTATE_IF(vbasedev->dev), "vfio", vbasedev);
vfio_migration_free(vbasedev);
vfio_unblock_multiple_devices_migration();
vfio_migration_deinit(vbasedev);
}

if (vbasedev->migration_blocker) {
Expand Down
10 changes: 8 additions & 2 deletions hw/vfio/pci-quirks.c
View file
Expand Up @@ -1530,6 +1530,12 @@ const PropertyInfo qdev_prop_nv_gpudirect_clique = {
.set = set_nv_gpudirect_clique_id,
};

static bool is_valid_std_cap_offset(uint8_t pos)
{
return (pos >= PCI_STD_HEADER_SIZEOF &&
pos <= (PCI_CFG_SPACE_SIZE - PCI_CAP_SIZEOF));
}

static int vfio_add_nv_gpudirect_cap(VFIOPCIDevice *vdev, Error **errp)
{
PCIDevice *pdev = &vdev->pdev;
Expand Down Expand Up @@ -1563,7 +1569,7 @@ static int vfio_add_nv_gpudirect_cap(VFIOPCIDevice *vdev, Error **errp)
*/
ret = pread(vdev->vbasedev.fd, &tmp, 1,
vdev->config_offset + PCI_CAPABILITY_LIST);
if (ret != 1 || !tmp) {
if (ret != 1 || !is_valid_std_cap_offset(tmp)) {
error_setg(errp, "NVIDIA GPUDirect Clique ID: error getting cap list");
return -EINVAL;
}
Expand All @@ -1575,7 +1581,7 @@ static int vfio_add_nv_gpudirect_cap(VFIOPCIDevice *vdev, Error **errp)
d4_conflict = true;
}
tmp = pdev->config[tmp + PCI_CAP_LIST_NEXT];
} while (tmp);
} while (is_valid_std_cap_offset(tmp));

if (!c8_conflict) {
pos = 0xC8;
Expand Down

0 comments on commit fcb237e

Please sign in to comment.