iommufd: Add vfio_group_set/unset_iommufd helpers

IOMMUFD has VFIO compatibility so user space can open /dev/iommu
to reuse existing VFIO IOMMU ioctls. Typically, /dev/vfio/vfio can
be a symlink to /dev/iommu. However, the group_fd used for
VFIO_GROUP_SET/UNSET_CONTAINER is still a VFIO thing and is not
supported using new IOMMUFD ioctls, which go for an approach of
attaching and dettaching a VFIO device directly instead of a VFIO
group.

In order to fill this gap, this patch adds a pair of helpers for
VFIO to call, so as to bind/unbind all devices in the VFIO group
to/from the iommufd and to attach/detach them to/from the ioas.

Signed-off-by: Nicolin Chen <nicolinc@nvidia.com>
Signed-off-by: Yi Liu <yi.l.liu@intel.com>

drivers/iommu/iommufd/iommufd_private.h

@@ -69,6 +69,7 @@ struct iommufd_ctx {
 	struct xarray objects;
 
 	struct iommufd_ioas *vfio_ioas;
+	int vfio_fd;
 };
 
 struct iommufd_ctx *iommufd_fget(int fd);

drivers/iommu/iommufd/vfio_compat.c

@@ -28,9 +28,8 @@ static struct iommufd_ioas *get_compat_ioas(struct iommufd_ctx *ictx)
 /*
  * Only attaching a group should cause a default creation of the internal ioas,
  * this returns the existing ioas if it has already been assigned somehow
- * FIXME: maybe_unused
  */
-static __maybe_unused struct iommufd_ioas *
+static struct iommufd_ioas *
 create_compat_ioas(struct iommufd_ctx *ictx)
 {
 	struct iommufd_ioas *ioas = NULL;
@@ -399,3 +398,103 @@ int iommufd_vfio_ioctl(struct iommufd_ctx *ictx, unsigned int cmd,
 	}
 	return -ENOIOCTLCMD;
 }
+
+#define vfio_device_detach_unbind(ictx, id, device, device_list)               \
+	({                                                                     \
+		list_for_each_entry(device, device_list, group_next) {         \
+			struct vfio_device_detach_ioas detach = {            \
+				.argsz = sizeof(detach),                       \
+				.ioas_id = id,                               \
+				.iommufd = ictx->vfio_fd,                      \
+			};                                                     \
+			if (device->ops->detach_ioas)                        \
+				device->ops->detach_ioas(device, &detach);   \
+			if (device->ops->unbind_iommufd)                       \
+				device->ops->unbind_iommufd(device);           \
+		}                                                              \
+	})
+
+struct iommufd_ctx *vfio_group_set_iommufd(int fd, struct list_head *device_list)
+{
+	struct vfio_device_attach_ioas attach = { .argsz = sizeof(attach) };
+	struct vfio_device_bind_iommufd bind = { .argsz = sizeof(bind) };
+	struct iommufd_ctx *ictx = iommufd_fget(fd);
+	struct iommufd_ioas *ioas;
+	struct vfio_device *device;
+	int rc;
+
+	if (!ictx)
+		return ictx;
+
+	ictx->vfio_fd = fd;
+
+	/*
+	 * Note: bind.dev_cookie is designed for page fault, whose uAPI is TBD
+	 * on IOMMUFD. And VFIO does not support that either. So we here leave
+	 * the dev_cookie to 0 for now, until it is available in VFIO too.
+	 */
+	bind.dev_cookie = 0;
+	bind.iommufd = fd;
+	attach.iommufd = fd;
+
+	ioas = create_compat_ioas(ictx);
+	if (IS_ERR(ioas))
+		goto out_fput;
+
+	xa_lock(&ictx->objects);
+	ictx->vfio_ioas = ioas;
+	attach.ioas_id = ioas->obj.id;
+	xa_unlock(&ictx->objects);
+
+	iommufd_put_object(&ioas->obj);
+
+	list_for_each_entry(device, device_list, group_next) {
+		if (!device->ops->bind_iommufd || !device->ops->unbind_iommufd)
+			goto cleanup_ioas;
+
+		rc = device->ops->bind_iommufd(device, &bind);
+		if (rc)
+			goto cleanup_ioas;
+
+		if (unlikely(!device->ops->attach_ioas))
+			goto cleanup_ioas;
+
+		rc = device->ops->attach_ioas(device, &attach);
+		if (rc)
+			goto cleanup_ioas;
+	}
+
+	return ictx;
+
+cleanup_ioas:
+	vfio_device_detach_unbind(ictx, attach.ioas_id, device, device_list);
+	iommufd_ioas_destroy(&ioas->obj);
+out_fput:
+	fput(ictx->filp);
+
+	return NULL;
+}
+EXPORT_SYMBOL_GPL(vfio_group_set_iommufd);
+
+void vfio_group_unset_iommufd(void *iommufd, struct list_head *device_list)
+{
+	struct iommufd_ctx *ictx = (struct iommufd_ctx *)iommufd;
+	struct iommufd_ioas *ioas;
+	struct vfio_device *device;
+	unsigned int ioas_id;
+
+	if (!ictx)
+		return;
+
+	ioas = get_compat_ioas(ictx);
+	if (IS_ERR(ioas))
+		return;
+
+	ioas_id = ioas->obj.id;
+	iommufd_put_object(&ioas->obj);
+
+	vfio_device_detach_unbind(ictx, ioas_id, device, device_list);
+	iommufd_ioas_destroy(&ioas->obj);
+	fput(ictx->filp);
+}
+EXPORT_SYMBOL_GPL(vfio_group_unset_iommufd);

include/linux/iommufd.h

@@ -15,6 +15,7 @@
 
 struct pci_dev;
 struct iommufd_device;
+struct iommufd_ctx;
 
 #if IS_ENABLED(CONFIG_IOMMUFD)
 struct iommufd_device *iommufd_bind_pci_device(int fd, struct pci_dev *pdev,
@@ -28,6 +29,8 @@ int iommufd_device_attach(struct iommufd_device *idev, u32 *pt_id,
 			  unsigned int flags);
 void iommufd_device_detach(struct iommufd_device *idev);
 
+struct iommufd_ctx *vfio_group_set_iommufd(int fd, struct list_head *device_list);
+void vfio_group_unset_iommufd(void *iommufd, struct list_head *device_list);
 #else /* !CONFIG_IOMMUFD */
 static inline struct iommufd_device *
 iommufd_bind_pci_device(int fd, struct pci_dev *pdev, u32 *id)
@@ -48,5 +51,16 @@ static inline int iommufd_device_attach(struct iommufd_device *idev,
 static inline void iommufd_device_detach(struct iommufd_device *idev)
 {
 }
+
+static inline struct iommufd_ctx *
+vfio_group_set_iommufd(int fd, struct list_head *device_list)
+{
+	return NULL;
+}
+
+static inline void vfio_group_unset_iommufd(void *iommufd,
+					    struct list_head *device_list)
+{
+}
 #endif /* CONFIG_IOMMUFD */
 #endif