virtio-mmio: hypervisor-less mode

Added support for hypervisor-less virtio.
- HVP dispatch function table
- Semaphore-based configuration synchronization mechanism when in
yield-capable context

Signed-off-by: Dan Milea <dan.milea@windriver.com>

cmake/options.cmake

@@ -80,11 +80,17 @@ endif (NOT WITH_VIRTIO_DEVICE AND NOT WITH_VIRTIO_SLAVE)
 
 option (WITH_VIRTIO_MMIO_DRV "Build with virtio mmio driver support enabled" OFF)
 option (WITH_VIRTIO_MMIO_DEV "Build with virtio mmio device support enabled" OFF)
+option (WITH_HVL_VIRTIO_DRV "Build with hypervisor-less virtio (front end) enabled" OFF)
 
 if (WITH_VIRTIO_MMIO_DRV)
   add_definitions(-DWITH_VIRTIO_MMIO_DRV)
 endif (WITH_VIRTIO_MMIO_DRV)
 
+if (WITH_HVL_VIRTIO_DRV)
+  set (WITH_VIRTIO_MMIO_DRV ON)
+  add_definitions(-DHVL_VIRTIO)
+endif (WITH_HVL_VIRTIO_DRV)
+
 if (WITH_VIRTIO_MMIO_DEV)
   add_definitions(-DWITH_VIRTIO_MMIO_DEV)
 endif (WITH_VIRTIO_MMIO_DEV)

lib/include/openamp/virtio_mmio.h

@@ -12,7 +12,10 @@
 #include <metal/device.h>
 #include <openamp/virtio.h>
 #include <openamp/virtqueue.h>
-
+#if defined(HVL_VIRTIO)
+#include <metal/mutex.h>
+#include <openamp/virtio_mmio_hvl.h>
+#endif
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -166,6 +169,26 @@ struct virtio_mmio_device {
 
 	/** Custom user data */
 	void *user_data;
+
+#if defined(HVL_VIRTIO)
+	/** Hypervisorless virtio mode flag */
+	unsigned int hvl_mode;
+
+	/** Hypervisorless virtio bounce buffer list */
+	struct metal_list bounce_buf_list;
+
+	/** Hypervisorless virtio inter-processor notification routine */
+	virtio_mmio_hvl_ipi_t ipi;
+
+	/** Hypervisorless virtio inter-processor notification routine parameter */
+	void *ipi_param;
+
+	/** Configuration synchronization semaphore usable in yield context */
+	metal_mutex_t cfg_sem;
+
+	/** Hypervisorless virtio dispatch table */
+	struct hvl_dispatch *hvl_func;
+#endif
 };
 
 /**

lib/include/openamp/virtio_mmio_hvl.h

@@ -0,0 +1,178 @@
+/*
+ * Copyright (c) 2022 Wind River Systems, Inc.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#ifndef OPENAMP_VIRTIO_MMIO_HVL_H
+#define OPENAMP_VIRTIO_MMIO_HVL_H
+
+#include <openamp/virtqueue.h>
+#include <openamp/virtio.h>
+#include <metal/device.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#if defined(HVL_VIRTIO)
+
+/*
+ * To minimize the number of hardware notification channels used, in hypervirsorless virtio
+ * deployments, a single multiplexed source of notifications is used for all the devices.
+ * The callback framework allows individual device drivers, if needed, to register their
+ * own callbacks (i.e. the equivalent of per-device ISRs)
+ */
+
+/* Max number of hypervisorless virtio callbacks */
+#define VIRTIO_MMIO_HVL_CB_MAX 10
+
+/*
+ * For multi-virtqueue devices in hypervisorless mode, a custom configuration acknowledgment
+ * mechanism is used to process same-register configuration entries like QUEUE_PFN. This is
+ * the ACK value from the physical machine monitor for these configurations.
+ */
+#define VIRTIO_MMIO_HVL_CFG_ACK 0xAABBAABB
+
+/* Hypervisorless virtio callback type */
+typedef void (*virtio_mmio_hvl_cb_t)(void *);
+
+/* Hypervisorless virtio inter-processor notification routine type */
+typedef void (*virtio_mmio_hvl_ipi_t)(void *);
+
+struct hvl_dispatch {
+	/**
+	 * @brief VIRTIO MMIO shared memory pool initialization routine.
+	 *
+	 * @param mem Pointer to memory.
+	 * @param size Size of memory region in bytes.
+	 */
+	void (*hvl_shm_pool_init)(void *mem, size_t size);
+
+	/**
+	 * @brief VIRTIO MMIO shared memory pool buffer allocation routine.
+	 *
+	 * @param size Number of bytes requested.
+	 *
+	 * @return pointer to allocated memory space in shared memory region.
+	 */
+	void* (*hvl_shm_pool_alloc)(size_t size);
+
+	/**
+	 * @brief VIRTIO MMIO shared memory pool buffer deallocation routine.
+	 *
+	 * @param ptr Pointer to memory space to free.
+	 */
+	void (*hvl_shm_pool_free)(void *ptr);
+
+	/**
+	 * @brief Wait for configuration ack for device vdev; OS-specific implementation
+	 *
+	 * @param vdev Pointer to virtio device structure.
+	 * @param usec timeout in microseconds.
+	 */
+	void (*hvl_wait_cfg_event)(struct virtio_device *vdev, uint32_t usec);
+
+	/**
+	 * @brief VIRTIO MMIO (hypervisorless mode) inter-processor notification routine
+	 */
+	void (*hvl_ipi)(void *param);
+};
+
+struct vq_bounce_buf {
+	struct metal_list node;
+	void *cookie;
+	uint64_t addr;
+	uint32_t len;
+};
+
+/**
+ * @brief VIRTIO MMIO hypervisorless operation mode initialization
+ *
+ * @param vdev Pointer to struct virtio_device.
+ */
+
+void virtio_mmio_hvl_init(struct virtio_device *vdev);
+
+/**
+ * @brief VIRTIO MMIO shared memory pool initialization routine.
+ *
+ * @param mem Pointer to memory.
+ * @param size Size of memory region in bytes.
+ */
+
+/**
+ * @brief Add VIRTIO MMIO (hypervisorless mode) callback.
+ *
+ * @param func Callback function pointer.
+ * @param arg  Callback parameter.
+ *
+ * @return int 0 for success.
+ */
+
+int virtio_mmio_hvl_cb_set(virtio_mmio_hvl_cb_t func, void *arg);
+
+/**
+ * @brief VIRTIO MMIO (hypervisorless mode) callback execution routine.
+ */
+
+void virtio_mmio_hvl_cb_run(void);
+
+/**
+ * @brief Add VIRTIO MMIO (hypervisorless mode) shared-memory bounce buffer.
+ *
+ * In hypervisorless mode this routine transparently allocates a bounce buffer
+ * to be enqueued in vring for consumption, copies the data in the original
+ * buffer and  saves the original buffer information for dequeueing.
+ * If the buffer is already in the pre-shared memory region, the bouncing
+ * mechanism is not used.
+ *
+ * @param vq     Pointer to VirtIO queue control block.
+ * @param cookie Pointer to hold call back data
+ * @param buffer Data buffer
+ * @param len    Data buffer length
+ *
+ * @return bounce buffer physical address or METAL_BAD_PHYS
+ */
+
+uint64_t virtio_mmio_hvl_add_bounce_buf(struct virtqueue *vq, void *cookie, char *buffer,
+					unsigned int len);
+
+/**
+ * @brief Get used VIRTIO MMIO (hypervisorless mode) shared-memory bounce buffer.
+ *
+ * In hypervisorless mode this routine transparently copies data from a dequeued
+ * bounce buffer in pre-shared memory to the original buffer location.
+ *
+ * @param vq     Pointer to VirtIO queue control block.
+ * @param dp     Pointer to vring descriptor.
+ * @param cookie Pointer for callback data
+ */
+
+void virtio_mmio_hvl_get_bounce_buf(struct virtqueue *vq, struct vring_desc *dp,
+				    void *cookie);
+
+/**
+ * @brief Wait for configuration ack, i.e. value at offset to match data
+ *
+ * @return int 0 for success
+ */
+int virtio_mmio_hvl_wait_cfg(struct virtio_device *vdev, int offset, uint32_t data);
+
+/**
+ * @brief Set inter-processor notification routine
+ *
+ * @param vdev     Pointer to virtio_device structure
+ * @param ipi_func Inter-processor notification routine
+ * @param param    Inter-processor notification routine parameter
+ */
+void virtio_mmio_hvl_set_ipi(struct virtio_device *vdev, virtio_mmio_hvl_ipi_t ipi_func,
+			     void *param);
+
+#endif /* defined(HVL_VIRTIO) */
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* OPENAMP_VIRTIO_MMIO_HVL_H */

lib/virtio/virtqueue.c

@@ -11,7 +11,9 @@
 #include <metal/atomic.h>
 #include <metal/log.h>
 #include <metal/alloc.h>
-
+#if defined(HVL_VIRTIO)
+#include <openamp/virtio_mmio_hvl.h>
+#endif
 /* Prototype for internal functions. */
 static void vq_ring_init(struct virtqueue *, void *, int);
 static void vq_ring_update_avail(struct virtqueue *, uint16_t);
@@ -166,6 +168,12 @@ void *virtqueue_get_buffer(struct virtqueue *vq, uint32_t *len, uint16_t *idx)
 	vq_ring_free_chain(vq, desc_idx);
 
 	cookie = vq->vq_descx[desc_idx].cookie;
+
+#if defined(HVL_VIRTIO)
+	virtio_mmio_hvl_get_bounce_buf(vq, &vq->vq_ring.desc[desc_idx],
+				       &vq->vq_descx[desc_idx].cookie);
+#endif
+
 	vq->vq_descx[desc_idx].cookie = NULL;
 
 	if (idx)
@@ -403,6 +411,14 @@ static uint16_t vq_ring_add_buffer(struct virtqueue *vq,
 	struct vring_desc *dp;
 	int i, needed;
 	uint16_t idx;
+#if defined(HVL_VIRTIO)
+	uint64_t addr = 0;
+	struct vq_desc_extra *dxp = NULL;
+	void *cookie = NULL;
+
+	dxp = &vq->vq_descx[head_idx];
+	cookie = dxp->cookie;
+#endif
 
 	(void)vq;
 
@@ -428,6 +444,13 @@ static uint16_t vq_ring_add_buffer(struct virtqueue *vq,
 		if (i >= readable)
 			dp->flags |= VRING_DESC_F_WRITE;
 
+#if defined(HVL_VIRTIO)
+		addr = virtio_mmio_hvl_add_bounce_buf(vq, &dxp->cookie, buf_list[i].buf,
+						      buf_list[i].len);
+		if (addr != 0) {
+			dp->addr = addr;
+		}
+#endif
 		/*
 		 * Instead of flushing the whole desc region, we flush only the
 		 * single entry hopefully saving some cycles

lib/virtio_mmio/CMakeLists.txt

@@ -5,6 +5,10 @@ collect (PROJECT_LIB_SOURCES virtio_net_drv.c)
 collect (PROJECT_LIB_SOURCES virtio_serial_drv.c)
 endif (WITH_VIRTIO_MMIO_DRV)
 
+if (WITH_HVL_VIRTIO_DRV)
+collect (PROJECT_LIB_SOURCES virtio_mmio_hvl.c)
+endif (WITH_HVL_VIRTIO_DRV)
+
 if (WITH_VIRTIO_MMIO_DEV)
 collect (PROJECT_LIB_SOURCES virtio_mmio_dev.c)
 endif (WITH_VIRTIO_MMIO_DEV)

lib/virtio_mmio/virtio_mmio_drv.c

@@ -45,7 +45,17 @@ static inline uint8_t virtio_mmio_read8(struct virtio_device *vdev, int offset)
 
 static inline void virtio_mmio_set_status(struct virtio_device *vdev, uint8_t status)
 {
+#if defined(HVL_VIRTIO)
+	struct virtio_mmio_device *vmdev = metal_container_of(vdev,
+							      struct virtio_mmio_device, vdev);
+#endif
+
 	virtio_mmio_write32(vdev, VIRTIO_MMIO_STATUS, status);
+#if defined(HVL_VIRTIO)
+	if (vmdev && vmdev->ipi && vmdev->hvl_mode == 1) {
+		vmdev->ipi(vmdev->ipi_param);
+	}
+#endif
 }
 
 static uint8_t virtio_mmio_get_status(struct virtio_device *vdev)
@@ -98,6 +108,10 @@ static void _virtio_mmio_set_features(struct virtio_device *vdev,
 				      uint32_t features, int idx)
 {
 	uint32_t hfeatures;
+#if defined(HVL_VIRTIO)
+	struct virtio_mmio_device *vmdev = metal_container_of(vdev,
+							      struct virtio_mmio_device, vdev);
+#endif
 
 	/* Writing selection register VIRTIO_MMIO_DEVICE_FEATURES_SEL. In pure AMP
 	 * mode this needs to be followed by a synchronization w/ the device
@@ -108,6 +122,11 @@ static void _virtio_mmio_set_features(struct virtio_device *vdev,
 	features &= hfeatures;
 	virtio_mmio_write32(vdev, VIRTIO_MMIO_DRIVER_FEATURES, features);
 	vdev->features = features;
+#if defined(HVL_VIRTIO)
+	if (vmdev && vmdev->ipi && vmdev->hvl_mode == 1) {
+		vmdev->ipi(vmdev->ipi_param);
+	}
+#endif
 }
 
 static void virtio_mmio_set_features(struct virtio_device *vdev, uint32_t features)
@@ -120,10 +139,19 @@ static void virtio_mmio_reset_device(struct virtio_device *vdev)
 	virtio_mmio_set_status(vdev, 0);
 }
 
-static void virtio_mmio_notify(struct virtqueue *vq)
+static void virtio_mmio_drv_notify(struct virtqueue *vq)
 {
+#if defined(HVL_VIRTIO)
+	struct virtio_mmio_device *vmdev = metal_container_of(vq->vq_dev,
+							      struct virtio_mmio_device, vdev);
+#endif
 	/* VIRTIO_F_NOTIFICATION_DATA is not supported for now */
 	virtio_mmio_write32(vq->vq_dev, VIRTIO_MMIO_QUEUE_NOTIFY, vq->vq_queue_index);
+#if defined(HVL_VIRTIO)
+	if (vmdev && vmdev->ipi && vmdev->hvl_mode == 1) {
+		vmdev->ipi(vmdev->ipi_param);
+	}
+#endif
 }
 
 const struct virtio_dispatch virtio_mmio_dispatch = {
@@ -135,7 +163,7 @@ const struct virtio_dispatch virtio_mmio_dispatch = {
 	.read_config = virtio_mmio_read_config,
 	.write_config = virtio_mmio_write_config,
 	.reset_device = virtio_mmio_reset_device,
-	.notify = virtio_mmio_notify,
+	.notify = virtio_mmio_drv_notify,
 };
 
 static int virtio_mmio_get_metal_io(struct virtio_device *vdev, uintptr_t virt_mem_ptr,
@@ -200,6 +228,9 @@ int virtio_mmio_device_init(struct virtio_mmio_device *vmdev, uintptr_t virt_mem
 {
 	struct virtio_device *vdev = &vmdev->vdev;
 	uint32_t magic, version, devid, vendor;
+#if defined(HVL_VIRTIO)
+	uint64_t shm_base, shm_len;
+#endif
 
 	vdev->role = vmdev->device_mode;
 	vdev->priv = vmdev;
@@ -234,6 +265,25 @@ int virtio_mmio_device_init(struct virtio_mmio_device *vmdev, uintptr_t virt_mem
 	vdev->id.device = devid;
 	vdev->id.vendor = vendor;
 
+#if defined(HVL_VIRTIO)
+	/* Check shared memory information.
+	 * This is just a flag for enabling hypervisorless mode for now.
+	 */
+	shm_base = virtio_mmio_read32(vdev, VIRTIO_MMIO_SHM_BASE_HIGH);
+	shm_base <<= 32;
+	shm_base |= virtio_mmio_read32(vdev, VIRTIO_MMIO_SHM_BASE_LOW);
+	shm_len = virtio_mmio_read32(vdev, VIRTIO_MMIO_SHM_LEN_HIGH);
+	shm_len <<= 32;
+	shm_len |= virtio_mmio_read32(vdev, VIRTIO_MMIO_SHM_LEN_LOW);
+
+	if (shm_base != 0 && shm_len != 0) {
+		vmdev->hvl_mode = 1;
+		virtio_mmio_hvl_cb_set((virtio_mmio_hvl_cb_t)virtio_mmio_isr, vdev);
+		virtio_mmio_hvl_init(vdev);
+		/* Initialize bounce buffer list for hypervisorless virtio */
+		metal_list_init(&vmdev->bounce_buf_list);
+	}
+#endif
 	virtio_mmio_set_status(vdev, VIRTIO_CONFIG_STATUS_ACK);
 	virtio_mmio_write32(vdev, VIRTIO_MMIO_GUEST_PAGE_SIZE, 4096);
 
@@ -327,6 +377,24 @@ struct virtqueue *virtio_mmio_setup_virtqueue(struct virtio_device *vdev,
 	virtio_mmio_write32(vdev, VIRTIO_MMIO_QUEUE_PFN,
 			    ((uintptr_t)metal_io_virt_to_phys(vq->shm_io,
 			    (char *)vq->vq_ring.desc)) / 4096);
+#if defined(HVL_VIRTIO)
+		if (vmdev && vmdev->ipi && vmdev->hvl_mode == 1) {
+			vmdev->ipi(vmdev->ipi_param);
+			/*
+			 * For multi-virtqueue devices in hypervisorless mode, a custom
+			 * configuration acknowledgment mechanism is used to process
+			 * same-register configuration entries like QUEUE_PFN.
+			 */
+			if (virtio_mmio_hvl_wait_cfg(vdev, VIRTIO_MMIO_QUEUE_PFN,
+						     VIRTIO_MMIO_HVL_CFG_ACK) != 0) {
+				metal_log(METAL_LOG_ERROR, "HVL mode: configuration failed\n");
+				return NULL;
+			}
+			virtio_mmio_write32(vdev, VIRTIO_MMIO_QUEUE_PFN,
+					    ((uintptr_t)metal_io_virt_to_phys(vq->shm_io,
+					    (char *)vq->vq_ring.desc)) / 4096);
+		}
+#endif
 
 	vdev->vrings_info[vdev->vrings_num].vq = vq;
 	vdev->vrings_num++;
@@ -338,11 +406,25 @@ struct virtqueue *virtio_mmio_setup_virtqueue(struct virtio_device *vdev,
 void virtio_mmio_isr(struct virtio_device *vdev)
 {
 	struct virtio_vring_info *vrings_info = vdev->vrings_info;
-
+#if defined(HVL_VIRTIO)
+	struct virtio_mmio_device *vmdev = metal_container_of(vdev,
+							      struct virtio_mmio_device, vdev);
+#endif
 	uint32_t isr = virtio_mmio_read32(vdev, VIRTIO_MMIO_INTERRUPT_STATUS);
 	struct virtqueue *vq;
 	unsigned int i;
 
+#if defined(HVL_VIRTIO)
+	if (vmdev && vmdev->hvl_mode == 1) {
+		uint32_t val;
+
+		val = metal_io_read32(vmdev->cfg_io, VIRTIO_MMIO_QUEUE_PFN);
+		if (val == VIRTIO_MMIO_HVL_CFG_ACK) {
+			metal_mutex_release(&vmdev->cfg_sem);
+		}
+	}
+#endif
+
 	if (isr & VIRTIO_MMIO_INT_VRING) {
 		for (i = 0; i < vdev->vrings_num; i++) {
 			vq = vrings_info[i].vq;
@@ -355,6 +437,13 @@ void virtio_mmio_isr(struct virtio_device *vdev)
 		metal_log(METAL_LOG_WARNING, "Unhandled interrupt type: 0x%x\n", isr);
 
 	virtio_mmio_write32(vdev, VIRTIO_MMIO_INTERRUPT_ACK, isr);
+#if defined(HVL_VIRTIO)
+	if (isr != 0) {
+		if (vmdev && vmdev->ipi && vmdev->hvl_mode == 1) {
+			vmdev->ipi(vmdev->ipi_param);
+		}
+	}
+#endif
 }
 
 static int virtio_mmio_create_virtqueues(struct virtio_device *vdev, unsigned int flags,

lib/virtio_mmio/virtio_mmio_hvl.c

@@ -0,0 +1,202 @@
+/*
+ * Copyright (c) 2022 Wind River Systems, Inc.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ *
+ * $FreeBSD$
+ */
+
+#if defined(HVL_VIRTIO)
+
+#include <openamp/open_amp.h>
+#include <openamp/virtqueue.h>
+#include <openamp/virtio.h>
+#include <openamp/virtio_mmio.h>
+#include <openamp/virtio_mmio_hvl.h>
+#include <metal/device.h>
+#include <metal/mutex.h>
+#include <metal/sleep.h>
+
+/*
+ * The implementation depends on OS-specific services for memory pool management
+ * routines, blocking wait and IPI.
+ * TODO: Replace them in a future iteration of the hypervisorless virtio feature
+ * set with APIs (to be introduced) in libmetal.
+ */
+
+/* hypervisorless virtio callback framework */
+static virtio_mmio_hvl_cb_t _cb_funcs[VIRTIO_MMIO_HVL_CB_MAX];
+static void *_cb_args[VIRTIO_MMIO_HVL_CB_MAX];
+static int _num_cb_funcs;
+
+/* pre-shared memory */
+uint32_t __shmem0_pool_start __attribute__((weak));
+uint32_t __shmem0_end __attribute__((weak));
+
+void virtio_mmio_hvl_init(struct virtio_device *vdev)
+{
+	static int init;
+	uint32_t *shmem_pool_mem = NULL;
+	uint32_t shmem_pool_size = 0;
+	struct virtio_mmio_device *vmdev = metal_container_of(vdev,
+							      struct virtio_mmio_device, vdev);
+
+	VIRTIO_ASSERT(vmdev->hvl_func->hvl_shm_pool_init,
+		      "Invalid HVL dispatch configuration");
+	VIRTIO_ASSERT(vmdev->hvl_func->hvl_shm_pool_alloc,
+		      "Invalid HVL dispatch configuration");
+	VIRTIO_ASSERT(vmdev->hvl_func->hvl_shm_pool_free,
+		      "Invalid HVL dispatch configuration");
+	VIRTIO_ASSERT(vmdev->hvl_func->hvl_wait_cfg_event,
+		      "Invalid HVL dispatch configuration");
+	VIRTIO_ASSERT(vmdev->hvl_func->hvl_ipi,
+		      "Invalid HVL dispatch configuration");
+
+	if (!init) {
+		shmem_pool_mem = &__shmem0_pool_start;
+		if (shmem_pool_mem != 0) {
+			shmem_pool_size = ((uintptr_t)&__shmem0_end -
+					   (uintptr_t)&__shmem0_pool_start);
+			vmdev->hvl_func->hvl_shm_pool_init(shmem_pool_mem, shmem_pool_size);
+			init = 1;
+		}
+	}
+	if (vdev) {
+		virtio_mmio_hvl_set_ipi(vdev, vmdev->hvl_func->hvl_ipi, NULL);
+	}
+	metal_mutex_init(&vmdev->cfg_sem);
+	metal_mutex_acquire(&vmdev->cfg_sem);
+}
+
+int virtio_mmio_hvl_cb_set(virtio_mmio_hvl_cb_t func, void *arg)
+{
+	if (_num_cb_funcs < VIRTIO_MMIO_HVL_CB_MAX - 1) {
+		_cb_funcs[_num_cb_funcs] = func;
+		_cb_args[_num_cb_funcs] = arg;
+		_num_cb_funcs++;
+		return 0;
+	}
+	return -1;
+}
+
+void virtio_mmio_hvl_cb_run(void)
+{
+	int i = 0;
+
+	for (i = 0; i < _num_cb_funcs; i++) {
+		if (_cb_funcs[i]) {
+			_cb_funcs[i](_cb_args[i]);
+		}
+	}
+}
+
+int virtio_mmio_hvl_wait_cfg(struct virtio_device *vdev, int offset, uint32_t data)
+{
+	uint32_t val;
+	struct virtio_mmio_device *vmdev = metal_container_of(vdev,
+							      struct virtio_mmio_device, vdev);
+
+	if (!vmdev) {
+		return -1;
+	}
+
+	if (vmdev->hvl_mode == 0) {
+		return 0;
+	}
+
+	val = metal_io_read32(vmdev->cfg_io, offset);
+
+	if (val == data) {
+		return 0;
+	}
+
+	vmdev->hvl_func->hvl_wait_cfg_event(vdev, 4000000);
+
+	val = metal_io_read32(vmdev->cfg_io, offset);
+	if (val == data) {
+		return 0;
+	}
+
+	return -1;
+}
+
+uint64_t virtio_mmio_hvl_add_bounce_buf(struct virtqueue *vq, void *cookie, char *buffer,
+					unsigned int len)
+{
+	struct vq_bounce_buf *bb = NULL;
+	uint64_t addr = 0;
+	char *buf = NULL;
+	struct virtio_mmio_device *vmdev = NULL;
+
+	if (!vq || !cookie || !buffer || len == 0)
+		return 0;
+
+	vmdev = metal_container_of(vq->vq_dev, struct virtio_mmio_device, vdev);
+
+	if (vmdev && vmdev->hvl_mode == 1) {
+		/* check if buffer is already in pre-shared shared memory */
+		addr = metal_io_virt_to_phys(vq->shm_io, buffer);
+		if (addr != METAL_BAD_PHYS) {
+			return addr;
+		}
+
+		buf = vmdev->hvl_func->hvl_shm_pool_alloc(len);
+		if (buf) {
+			memcpy(buf, buffer, len);
+			addr = metal_io_virt_to_phys(vq->shm_io, buf);
+			bb = vmdev->hvl_func->hvl_shm_pool_alloc(sizeof(struct vq_bounce_buf));
+			if (bb) {
+				bb->addr = (uintptr_t)buffer;
+				bb->len = len;
+				bb->cookie = cookie;
+				metal_list_add_tail(&vmdev->bounce_buf_list, &bb->node);
+				return addr;
+			}
+			vmdev->hvl_func->hvl_shm_pool_free(buf);
+		}
+	}
+
+	return 0;
+}
+
+void virtio_mmio_hvl_get_bounce_buf(struct virtqueue *vq, struct vring_desc *dp, void *cookie)
+{
+	struct vq_bounce_buf *bb = NULL;
+	struct metal_list *node = NULL;
+	struct virtio_mmio_device *vmdev = NULL;
+
+	VIRTIO_ASSERT(vq, "Invalid virtqueue");
+	VIRTIO_ASSERT(dp, "Invalid vring descriptor");
+	VIRTIO_ASSERT(cookie, "Invalid cookie");
+
+	vmdev = metal_container_of(vq->vq_dev, struct virtio_mmio_device, vdev);
+
+	if (vmdev && vmdev->hvl_mode == 1) {
+		metal_list_for_each(&vmdev->bounce_buf_list, node) {
+			bb = metal_container_of(node, struct vq_bounce_buf, node);
+			if (bb->cookie == cookie) {
+				memcpy((void *)(uintptr_t)bb->addr,
+				       (void *)(metal_io_phys_to_virt(vq->shm_io, dp->addr)),
+				       bb->len);
+				metal_list_del(node);
+				vmdev->hvl_func->hvl_shm_pool_free(bb);
+				vmdev->hvl_func->hvl_shm_pool_free(
+					(void *)(metal_io_phys_to_virt(vq->shm_io, dp->addr)));
+				break;
+			}
+		}
+	}
+}
+
+void virtio_mmio_hvl_set_ipi(struct virtio_device *vdev, virtio_mmio_hvl_ipi_t ipi_func,
+			     void *param)
+{
+	struct virtio_mmio_device *vmdev = metal_container_of(vdev,
+							      struct virtio_mmio_device, vdev);
+
+	vmdev->ipi = ipi_func;
+	vmdev->ipi_param = param;
+}
+
+#endif /* defined(HVL_VIRTIO) */
+

lib/virtio_mmio/virtio_serial_drv.c

@@ -134,7 +134,15 @@ void virtio_serial_poll_out(const struct virtio_device *vdev, unsigned char out_
 	}
 
 	if (virtqueue_full(vq)) {
+#if !defined(HVL_VIRTIO)
 		virtqueue_get_buffer(vq, NULL, NULL);
+#else
+		void *cookie = virtqueue_get_buffer(vq, NULL, NULL);
+
+		while (!cookie) {
+			cookie = virtqueue_get_buffer(vq, NULL, NULL);
+		}
+#endif
 	}
 
 	if (virtqueue_full(vq)) {