virtio-mmio: hypervisor-less mode

Added support for hypervisor-less virtio.

Signed-off-by: Dan Milea <dan.milea@windriver.com>

cmake/options.cmake

@@ -84,11 +84,17 @@ if (NOT WITH_VIRTIO_DEVICE AND NOT WITH_VIRTIO_SLAVE)
 endif (NOT WITH_VIRTIO_DEVICE AND NOT WITH_VIRTIO_SLAVE)
 
 option (WITH_VIRTIO_MMIO "Build with virtio mmio (front end) enabled" ON)
+option (WITH_HVL_VIRTIO "Build with hypervisor-less virtio (front end) enabled" OFF)
 
 if (WITH_VIRTIO_MMIO)
   add_definitions(-DWITH_VIRTIO_MMIO)
 endif (WITH_VIRTIO_MMIO)
 
+if (WITH_HVL_VIRTIO)
+  set (WITH_VIRTIO_MMIO ON)
+  add_definitions(-DHVL_VIRTIO)
+endif (WITH_HVL_VIRTIO)
+
 option (WITH_DCACHE_VRINGS "Build with vrings cache operations enabled" OFF)
 
 if (WITH_DCACHE_VRINGS)

lib/include/openamp/virtio_mmio.h

@@ -11,7 +11,9 @@
 #include <openamp/virtqueue.h>
 #include <openamp/virtio.h>
 #include <metal/device.h>
-
+#if defined(HVL_VIRTIO)
+#include <openamp/virtio_mmio_hvl.h>
+#endif
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -161,6 +163,15 @@ static struct vring __vring_##name = { \
 		(n + 1) * sizeof(uint16_t) + align - 1) & ~(align - 1))), \
 }
 
+#if defined(HVL_VIRTIO)  /* vrings in individual sections */
+#define VRBUF_DECLARE(name, n, align, section_name) \
+static char __vrbuf_##name[VRING_SIZE(n, align)] __aligned(4096) \
+	__attribute((__section__(".shared.vring."#section_name)))
+#else
+#define VRBUF_DECLARE(name, n, align, section_name) \
+static char __vrbuf_##name[VRING_SIZE(n, align)] __aligned(4096)
+#endif
+
 /**
  * @endcond
  */
@@ -174,7 +185,7 @@ static struct vring __vring_##name = { \
  */
 
 #define VQ_DECLARE(name, n, align) \
-	static char __vrbuf_##name[VRING_SIZE(n, align)] __aligned(4096); \
+	VRBUF_DECLARE(name, n, align, name); \
 	static struct { \
 	struct virtqueue vq; \
 	struct vq_desc_extra extra[n]; \
@@ -229,6 +240,12 @@ struct virtio_mmio_device {
 	unsigned int device_mode;
 	unsigned int irq;
 	void *user_data;
+#if defined(HVL_VIRTIO)
+	unsigned int hvl_mode;
+	struct metal_list bounce_buf_list;
+	virtio_mmio_hvl_ipi_t ipi;
+	void *ipi_param;
+#endif
 };
 
 /**

lib/include/openamp/virtio_mmio_hvl.h

@@ -0,0 +1,190 @@
+/*
+ * Copyright (c) 2022 Wind River Systems, Inc.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#ifndef OPENAMP_VIRTIO_MMIO_HVL_H
+#define OPENAMP_VIRTIO_MMIO_HVL_H
+
+#include <openamp/virtqueue.h>
+#include <openamp/virtio.h>
+#include <metal/device.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#if defined(HVL_VIRTIO)
+
+/*
+ * To minimize the number of hardware notification channels used, in hypervirsorless virtio
+ * deployments, a single multiplexed source of notifications is used for all the devices.
+ * The callback framework allows individual device drivers, if needed, to register their
+ * own callbacks (i.e. the equivalent of per-device ISRs)
+ */
+
+/* Max number of hypervisorless virtio callbacks */
+#define VIRTIO_MMIO_HVL_CB_MAX 10
+
+/*
+ * For multi-virtqueue devices in hypervisorless mode, a custom configuration acknowledgment
+ * mechanism is used to process same-register configuration entries like QUEUE_PFN. This is
+ * the ACK value from the physical machine monitor for these configurations.
+ */
+#define VIRTIO_MMIO_HVL_CFG_ACK 0xAABBAABB
+
+/* Hypervisorless virtio callback type */
+typedef void (*virtio_mmio_hvl_cb_t)(void *);
+
+/* Hypervisorless virtio inter-processor notification routine type */
+typedef void (*virtio_mmio_hvl_ipi_t)(void *);
+
+struct vq_bounce_buf {
+	struct metal_list node;
+	void *cookie;
+	uint64_t addr;
+	uint32_t len;
+};
+
+/**
+ * @brief VIRTIO MMIO hypervisorless operation mode initialization
+ *
+ * @param[in] vdev Pointer to struct virtio_device.
+ *
+ * @return N/A.
+ */
+
+void virtio_mmio_hvl_init(struct virtio_device *vdev);
+
+/**
+ * @brief VIRTIO MMIO shared memory pool initialization routine.
+ *
+ * @param[in] mem Pointer to memory.
+ * @param[in] size Size of memory region in bytes.
+ *
+ * @return N/A.
+ */
+
+void virtio_mmio_shm_pool_init(void *mem, size_t size);
+
+/**
+ * @brief VIRTIO MMIO shared memory pool buffer allocation routine.
+ *
+ * @param[in] size Number of bytes requested.
+ *
+ * @return pointer to allocated memory space in shared memory region.
+ */
+
+void *virtio_mmio_shm_pool_alloc(size_t size);
+
+/**
+ * @brief VIRTIO MMIO shared memory pool buffer deallocation routine.
+ *
+ * @param[in] ptr Pointer to memory space to free.
+ *
+ * @return N/A.
+ */
+
+void virtio_mmio_shm_pool_free(void *ptr);
+
+/**
+ * @brief VIRTIO MMIO (hypervisorless mode) inter-processor notification routine
+ *
+ * @return N/A.
+ */
+
+void virtio_mmio_hvl_ipi(void);
+
+/**
+ * @brief VIRTIO MMIO (hypervisorless mode) wait routine.
+ *
+ * @param[in] usec Number of microseconds to wait.
+ *
+ * @return N/A.
+ */
+
+void virtio_mmio_hvl_wait(uint32_t usec);
+
+/**
+ * @brief Add VIRTIO MMIO (hypervisorless mode) callback.
+ *
+ * @param[in] func Callback function pointer.
+ * @param[in] arg  Callback parameter.
+ *
+ * @return int 0 for success.
+ */
+
+int virtio_mmio_hvl_cb_set(virtio_mmio_hvl_cb_t func, void *arg);
+
+/**
+ * @brief VIRTIO MMIO (hypervisorless mode) callback execution routine.
+ *
+ * @return N/A
+ */
+
+void virtio_mmio_hvl_cb_run(void);
+
+/**
+ * @brief Add VIRTIO MMIO (hypervisorless mode) shared-memory bounce buffer.
+ *
+ * In hypervisorless mode this routine transparently allocates a bounce buffer
+ * to be enqueued in vring for consumption, copies the data in the original
+ * buffer and  saves the original buffer information for dequeueing.
+ * If the buffer is already in the pre-shared memory region, the bouncing
+ * mechanism is not used.
+ *
+ * @param[in] vq     Pointer to VirtIO queue control block.
+ * @param[in] cookie Pointer to hold call back data
+ * @param[in] buffer Data buffer
+ * @param[in] len    Data buffer length
+ *
+ * @return bounce buffer physical address or METAL_BAD_PHYS
+ */
+
+uint64_t virtio_mmio_hvl_add_bounce_buf(struct virtqueue *vq, void *cookie, char *buffer,
+					unsigned int len);
+
+/**
+ * @brief Get used VIRTIO MMIO (hypervisorless mode) shared-memory bounce buffer.
+ *
+ * In hypervisorless mode this routine transparently copies data from a dequeued
+ * bounce buffer in pre-shared memory to the original buffer location.
+ *
+ * @param[in] vq     Pointer to VirtIO queue control block.
+ * @param[in] dp     Pointer to vring descriptor.
+ * @param[in] cookie Pointer for callback data
+ *
+ * @return N/A
+ */
+
+void virtio_mmio_hvl_get_bounce_buf(struct virtqueue *vq, struct vring_desc *dp,
+				    void *cookie);
+
+/**
+ * @brief Wait for configuration ack, i.e. value at offset to match data
+ *
+ * @return int 0 for success
+ */
+
+int virtio_mmio_hvl_wait_cfg(struct virtio_device *vdev, int offset, uint32_t data);
+
+/**
+ * @brief Set inter-processor notification routine
+ *
+ * @param[in] vdev     Pointer to virtio_device structure
+ * @param[in] ipi_func Inter-processor notification routine
+ * @param[in] param    Inter-processor notification routine parameter
+ *
+ * @return N/A
+ */
+void virtio_mmio_hvl_set_ipi(struct virtio_device *vdev, virtio_mmio_hvl_ipi_t ipi_func,
+			     void *param);
+
+#endif /* defined(HVL_VIRTIO) */
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* OPENAMP_VIRTIO_MMIO_HVL_H */

lib/virtio/virtqueue.c

@@ -12,6 +12,9 @@
 #include <metal/log.h>
 #include <metal/alloc.h>
 #include <metal/cache.h>
+#if defined(HVL_VIRTIO)
+#include <openamp/virtio_mmio_hvl.h>
+#endif
 
 /* Prototype for internal functions. */
 static void vq_ring_init(struct virtqueue *, void *, int);
@@ -211,6 +214,11 @@ void *virtqueue_get_buffer(struct virtqueue *vq, uint32_t *len, uint16_t *idx)
 	vq_ring_free_chain(vq, desc_idx);
 
 	cookie = vq->vq_descx[desc_idx].cookie;
+
+#if defined(HVL_VIRTIO)
+	virtio_mmio_hvl_get_bounce_buf(vq, &vq->vq_ring.desc[desc_idx], cookie);
+#endif
+
 	vq->vq_descx[desc_idx].cookie = NULL;
 
 	if (idx)
@@ -523,6 +531,14 @@ static uint16_t vq_ring_add_buffer(struct virtqueue *vq,
 	struct vring_desc *dp;
 	int i, needed;
 	uint16_t idx;
+#if defined(HVL_VIRTIO)
+	uint64_t addr = 0;
+	struct vq_desc_extra *dxp = NULL;
+	void *cookie = NULL;
+
+	dxp = &vq->vq_descx[head_idx];
+	cookie = dxp->cookie;
+#endif
 
 	(void)vq;
 
@@ -548,6 +564,12 @@ static uint16_t vq_ring_add_buffer(struct virtqueue *vq,
 		if (i >= readable)
 			dp->flags |= VRING_DESC_F_WRITE;
 
+#if defined(HVL_VIRTIO)
+		addr = virtio_mmio_hvl_add_bounce_buf(vq, cookie, buf_list[i].buf, buf_list[i].len);
+		if (addr != 0) {
+			dp->addr = addr;
+		}
+#endif
 		/*
 		 * Instead of flushing the whole desc region, we flush only the
 		 * single entry hopefully saving some cycles

lib/virtio_mmio/CMakeLists.txt

@@ -1,4 +1,5 @@
 collect (PROJECT_LIB_SOURCES virtio_mmio.c)
+collect (PROJECT_LIB_SOURCES virtio_mmio_hvl.c)
 collect (PROJECT_LIB_SOURCES virtio_rng.c)
 collect (PROJECT_LIB_SOURCES virtio_net.c)
 collect (PROJECT_LIB_SOURCES virtio_serial.c)