vduse: Introduce VDUSE - vDPA Device in Userspace

This VDUSE driver enables implementing vDPA devices in userspace.
Both control path and data path of vDPA devices will be able to
be handled in userspace.

In the control path, the VDUSE driver will make use of message
mechnism to forward the actions (get/set features, get/st status,
get/set config space and set virtqueue states) from virtio-vdpa
driver to userspace. Userspace can use read()/write() to receive/reply
to those control messages.

In the data path, the VDUSE driver implements a MMU-based
on-chip IOMMU driver which supports both direct mapping and
indirect mapping with bounce buffer. Userspace can access those
iova space via mmap(). Besides, eventfd mechnism is used to
trigger interrupts and forward virtqueue kicks.

Signed-off-by: Xie Yongji <xieyongji@bytedance.com>

drivers/vdpa/Kconfig

@@ -19,6 +19,14 @@ config VDPA_SIM
 	  to RX. This device is used for testing, prototyping and
 	  development of vDPA.
 
+config VDPA_USER
+	tristate "VDUSE (vDPA Device in Userspace) support"
+	depends on EVENTFD && MMU && HAS_DMA
+	default n
+	help
+	  With VDUSE it is possible to emulate a vDPA Device
+	  in a userspace program.
+
 config IFCVF
 	tristate "Intel IFC VF vDPA driver"
 	depends on PCI_MSI

drivers/vdpa/Makefile

@@ -1,5 +1,6 @@
 # SPDX-License-Identifier: GPL-2.0
 obj-$(CONFIG_VDPA) += vdpa.o
 obj-$(CONFIG_VDPA_SIM) += vdpa_sim/
+obj-$(CONFIG_VDPA_USER) += vdpa_user/
 obj-$(CONFIG_IFCVF)    += ifcvf/
 obj-$(CONFIG_MLX5_VDPA) += mlx5/

drivers/vdpa/vdpa_user/Makefile

@@ -0,0 +1,5 @@
+# SPDX-License-Identifier: GPL-2.0
+
+vduse-y := vduse_dev.o iova_domain.o eventfd.o
+
+obj-$(CONFIG_VDPA_USER) += vduse.o

drivers/vdpa/vdpa_user/eventfd.c

@@ -0,0 +1,221 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Eventfd support for VDUSE
+ *
+ * Copyright (C) 2020 Bytedance Inc. and/or its affiliates. All rights reserved.
+ *
+ * Author: Xie Yongji <xieyongji@bytedance.com>
+ *
+ */
+
+#include <linux/eventfd.h>
+#include <linux/poll.h>
+#include <linux/wait.h>
+#include <linux/slab.h>
+#include <linux/file.h>
+#include <uapi/linux/vduse.h>
+
+#include "eventfd.h"
+
+static struct workqueue_struct *vduse_irqfd_cleanup_wq;
+
+static void vduse_virqfd_shutdown(struct work_struct *work)
+{
+	u64 cnt;
+	struct vduse_virqfd *virqfd = container_of(work,
+					struct vduse_virqfd, shutdown);
+
+	eventfd_ctx_remove_wait_queue(virqfd->ctx, &virqfd->wait, &cnt);
+	flush_work(&virqfd->inject);
+	eventfd_ctx_put(virqfd->ctx);
+	kfree(virqfd);
+}
+
+static void vduse_virqfd_inject(struct work_struct *work)
+{
+	struct vduse_virqfd *virqfd = container_of(work,
+					struct vduse_virqfd, inject);
+	struct vduse_virtqueue *vq = virqfd->vq;
+
+	spin_lock_irq(&vq->irq_lock);
+	if (vq->ready && vq->cb)
+		vq->cb(vq->private);
+	spin_unlock_irq(&vq->irq_lock);
+}
+
+static void virqfd_deactivate(struct vduse_virqfd *virqfd)
+{
+	queue_work(vduse_irqfd_cleanup_wq, &virqfd->shutdown);
+}
+
+static int vduse_virqfd_wakeup(wait_queue_entry_t *wait, unsigned int mode,
+				int sync, void *key)
+{
+	struct vduse_virqfd *virqfd = container_of(wait, struct vduse_virqfd, wait);
+	struct vduse_virtqueue *vq = virqfd->vq;
+
+	__poll_t flags = key_to_poll(key);
+
+	if (flags & EPOLLIN)
+		schedule_work(&virqfd->inject);
+
+	if (flags & EPOLLHUP) {
+		spin_lock(&vq->irq_lock);
+		if (vq->virqfd == virqfd) {
+			vq->virqfd = NULL;
+			virqfd_deactivate(virqfd);
+		}
+		spin_unlock(&vq->irq_lock);
+	}
+
+	return 0;
+}
+
+static void vduse_virqfd_ptable_queue_proc(struct file *file,
+			wait_queue_head_t *wqh, poll_table *pt)
+{
+	struct vduse_virqfd *virqfd = container_of(pt, struct vduse_virqfd, pt);
+
+	add_wait_queue(wqh, &virqfd->wait);
+}
+
+int vduse_virqfd_setup(struct vduse_dev *dev,
+			struct vduse_vq_eventfd *eventfd)
+{
+	struct vduse_virqfd *virqfd;
+	struct fd irqfd;
+	struct eventfd_ctx *ctx;
+	struct vduse_virtqueue *vq;
+	__poll_t events;
+	int ret;
+
+	if (eventfd->index >= dev->vq_num)
+		return -EINVAL;
+
+	vq = &dev->vqs[eventfd->index];
+	virqfd = kzalloc(sizeof(*virqfd), GFP_KERNEL);
+	if (!virqfd)
+		return -ENOMEM;
+
+	INIT_WORK(&virqfd->shutdown, vduse_virqfd_shutdown);
+	INIT_WORK(&virqfd->inject, vduse_virqfd_inject);
+
+	ret = -EBADF;
+	irqfd = fdget(eventfd->fd);
+	if (!irqfd.file)
+		goto err_fd;
+
+	ctx = eventfd_ctx_fileget(irqfd.file);
+	if (IS_ERR(ctx)) {
+		ret = PTR_ERR(ctx);
+		goto err_ctx;
+	}
+
+	virqfd->vq = vq;
+	virqfd->ctx = ctx;
+	spin_lock(&vq->irq_lock);
+	if (vq->virqfd)
+		virqfd_deactivate(virqfd);
+	vq->virqfd = virqfd;
+	spin_unlock(&vq->irq_lock);
+
+	init_waitqueue_func_entry(&virqfd->wait, vduse_virqfd_wakeup);
+	init_poll_funcptr(&virqfd->pt, vduse_virqfd_ptable_queue_proc);
+
+	events = vfs_poll(irqfd.file, &virqfd->pt);
+
+	/*
+	 * Check if there was an event already pending on the eventfd
+	 * before we registered and trigger it as if we didn't miss it.
+	 */
+	if (events & EPOLLIN)
+		schedule_work(&virqfd->inject);
+
+	fdput(irqfd);
+
+	return 0;
+err_ctx:
+	fdput(irqfd);
+err_fd:
+	kfree(virqfd);
+	return ret;
+}
+
+void vduse_virqfd_release(struct vduse_dev *dev)
+{
+	int i;
+
+	for (i = 0; i < dev->vq_num; i++) {
+		struct vduse_virtqueue *vq = &dev->vqs[i];
+
+		spin_lock(&vq->irq_lock);
+		if (vq->virqfd) {
+			virqfd_deactivate(vq->virqfd);
+			vq->virqfd = NULL;
+		}
+		spin_unlock(&vq->irq_lock);
+	}
+	flush_workqueue(vduse_irqfd_cleanup_wq);
+}
+
+int vduse_virqfd_init(void)
+{
+	vduse_irqfd_cleanup_wq = alloc_workqueue("vduse-irqfd-cleanup",
+						WQ_UNBOUND, 0);
+	if (!vduse_irqfd_cleanup_wq)
+		return -ENOMEM;
+
+	return 0;
+}
+
+void vduse_virqfd_exit(void)
+{
+	destroy_workqueue(vduse_irqfd_cleanup_wq);
+}
+
+void vduse_vq_kick(struct vduse_virtqueue *vq)
+{
+	spin_lock(&vq->kick_lock);
+	if (vq->ready && vq->kickfd)
+		eventfd_signal(vq->kickfd, 1);
+	spin_unlock(&vq->kick_lock);
+}
+
+int vduse_kickfd_setup(struct vduse_dev *dev,
+			struct vduse_vq_eventfd *eventfd)
+{
+	struct eventfd_ctx *ctx;
+	struct vduse_virtqueue *vq;
+
+	if (eventfd->index >= dev->vq_num)
+		return -EINVAL;
+
+	vq = &dev->vqs[eventfd->index];
+	ctx = eventfd_ctx_fdget(eventfd->fd);
+	if (IS_ERR(ctx))
+		return PTR_ERR(ctx);
+
+	spin_lock(&vq->kick_lock);
+	if (vq->kickfd)
+		eventfd_ctx_put(vq->kickfd);
+	vq->kickfd = ctx;
+	spin_unlock(&vq->kick_lock);
+
+	return 0;
+}
+
+void vduse_kickfd_release(struct vduse_dev *dev)
+{
+	int i;
+
+	for (i = 0; i < dev->vq_num; i++) {
+		struct vduse_virtqueue *vq = &dev->vqs[i];
+
+		spin_lock(&vq->kick_lock);
+		if (vq->kickfd) {
+			eventfd_ctx_put(vq->kickfd);
+			vq->kickfd = NULL;
+		}
+		spin_unlock(&vq->kick_lock);
+	}
+}

drivers/vdpa/vdpa_user/eventfd.h

@@ -0,0 +1,48 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Eventfd support for VDUSE
+ *
+ * Copyright (C) 2020 Bytedance Inc. and/or its affiliates. All rights reserved.
+ *
+ * Author: Xie Yongji <xieyongji@bytedance.com>
+ *
+ */
+
+#ifndef _VDUSE_EVENTFD_H
+#define _VDUSE_EVENTFD_H
+
+#include <linux/eventfd.h>
+#include <linux/poll.h>
+#include <linux/wait.h>
+#include <uapi/linux/vduse.h>
+
+#include "vduse.h"
+
+struct vduse_dev;
+
+struct vduse_virqfd {
+	struct eventfd_ctx *ctx;
+	struct vduse_virtqueue *vq;
+	struct work_struct inject;
+	struct work_struct shutdown;
+	wait_queue_entry_t wait;
+	poll_table pt;
+};
+
+int vduse_virqfd_setup(struct vduse_dev *dev,
+			struct vduse_vq_eventfd *eventfd);
+
+void vduse_virqfd_release(struct vduse_dev *dev);
+
+int vduse_virqfd_init(void);
+
+void vduse_virqfd_exit(void);
+
+void vduse_vq_kick(struct vduse_virtqueue *vq);
+
+int vduse_kickfd_setup(struct vduse_dev *dev,
+			struct vduse_vq_eventfd *eventfd);
+
+void vduse_kickfd_release(struct vduse_dev *dev);
+
+#endif /* _VDUSE_EVENTFD_H */