drivers/vhost: add vhost framework for NuttX

virtio is a framework to implement the virtio drivers
vhost (Virtual Host) is a framework to implement the virtio devices
With the virtio and vhost framework, we can use the virtio drivers
and vhost drivers to implement the cross-core communication.

Signed-off-by: Bowen Wang <wangbowen6@xiaomi.com>

Kconfig

@@ -2228,6 +2228,38 @@ config DEBUG_VIRTIO_INFO
 
 endif # DEBUG_VIDEO
 
+config DEBUG_VHOST
+	bool "Vhost Debug Features"
+	default n
+	depends on DRIVERS_VHOST
+	---help---
+		Enable vhost debug features.
+
+if DEBUG_VHOST
+
+config DEBUG_VHOST_ERROR
+	bool "Vhost Error Output"
+	default n
+	depends on DEBUG_ERROR
+	---help---
+		Enable vhost error output to SYSLOG.
+
+config DEBUG_VHOST_WARN
+	bool "Vhost Warnings Output"
+	default n
+	depends on DEBUG_WARN
+	---help---
+		Enable vhost warning output to SYSLOG.
+
+config DEBUG_VHOST_INFO
+	bool "Vhost Informational Output"
+	default n
+	depends on DEBUG_INFO
+	---help---
+		Enable vhost informational output to SYSLOG.
+
+endif # DEBUG_VHOST
+
 config DEBUG_RESET
 	bool "RESET Debug Features"
 	default n

drivers/Kconfig

@@ -19,6 +19,7 @@ source "drivers/ipcc/Kconfig"
 source "drivers/timers/Kconfig"
 source "drivers/analog/Kconfig"
 source "drivers/audio/Kconfig"
+source "drivers/vhost/Kconfig"
 source "drivers/video/Kconfig"
 source "drivers/virtio/Kconfig"
 source "drivers/bch/Kconfig"

drivers/Makefile

@@ -70,6 +70,7 @@ include usbdev/Make.defs
 include usbhost/Make.defs
 include usbmisc/Make.defs
 include usbmonitor/Make.defs
+include vhost/Make.defs
 include video/Make.defs
 include virtio/Make.defs
 include wireless/Make.defs

drivers/drivers_initialize.c

@@ -50,6 +50,7 @@
 #include <nuttx/thermal.h>
 #include <nuttx/trace.h>
 #include <nuttx/usrsock/usrsock_rpmsg.h>
+#include <nuttx/vhost/vhost.h>
 #include <nuttx/virtio/virtio.h>
 #include <nuttx/drivers/optee.h>
 
@@ -273,6 +274,10 @@ void drivers_initialize(void)
   virtio_register_drivers();
 #endif
 
+#ifdef CONFIG_DRIVERS_VHOST
+  vhost_register_drivers();
+#endif
+
 #ifndef CONFIG_DEV_OPTEE_NONE
   optee_register();
 #endif

drivers/vhost/CMakeLists.txt

@@ -0,0 +1,27 @@
+# ##############################################################################
+# drivers/vhost/CMakeLists.txt
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more contributor
+# license agreements.  See the NOTICE file distributed with this work for
+# additional information regarding copyright ownership.  The ASF licenses this
+# file to you under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License.  You may obtain a copy of
+# the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+# License for the specific language governing permissions and limitations under
+# the License.
+#
+# ##############################################################################
+set(SRCS)
+
+if(CONFIG_DRIVERS_VHOST)
+  list(APPEND SRCS vhost.c)
+endif()
+
+target_sources(drivers PRIVATE ${SRCS})
+target_include_directories(drivers PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})

drivers/vhost/Kconfig

@@ -0,0 +1,13 @@
+#
+# For a description of the syntax of this configuration file,
+# see the file kconfig-language.txt in the NuttX tools repository.
+#
+
+menuconfig DRIVERS_VHOST
+	bool "Virtual Host Support"
+	select OPENAMP
+	default n
+
+if DRIVERS_VHOST
+
+endif # DRIVERS_VHOST

drivers/vhost/Make.defs

@@ -0,0 +1,30 @@
+############################################################################
+# drivers/vhost/Make.defs
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.  The
+# ASF licenses this file to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance with the
+# License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+############################################################################
+
+# Include vhost support
+
+ifeq ($(CONFIG_DRIVERS_VHOST),y)
+  CSRCS += vhost.c
+endif
+
+# Include build support
+
+DEPPATH += --dep-path vhost
+VPATH += :vhost

drivers/vhost/vhost.c

@@ -0,0 +1,378 @@
+/****************************************************************************
+ * drivers/vhost/vhost.c
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.  The
+ * ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ ****************************************************************************/
+
+/****************************************************************************
+ * Included Files
+ ****************************************************************************/
+
+#include <debug.h>
+
+#include <nuttx/kmalloc.h>
+#include <nuttx/mutex.h>
+#include <nuttx/wqueue.h>
+#include <nuttx/vhost/vhost.h>
+
+/****************************************************************************
+ * Pre-processor Definitions
+ ****************************************************************************/
+
+#define VHOST_DEFERED_PROBE_PERIOD 100
+
+/****************************************************************************
+ * Private Types
+ ****************************************************************************/
+
+struct vhost_bus_s
+{
+  mutex_t          lock;           /* Lock for the list */
+  struct list_node device;         /* Wait match vhost device list */
+  struct list_node defered_device; /* Defered vhost device list */
+  struct list_node driver;         /* Vhost driver list */
+  struct work_s    defered_work;   /* Defered probe work */
+};
+
+struct vhost_device_item_s
+{
+  struct list_node     node;    /* List node */
+  struct vhost_device *device;  /* Pointer to the vhost device */
+  struct vhost_driver *driver;  /* Pointer to the vhost driver that
+                                 * matched with current vhost device
+                                 */
+};
+
+/****************************************************************************
+ * Private Data
+ ****************************************************************************/
+
+static struct vhost_bus_s g_vhost_bus =
+{
+  NXMUTEX_INITIALIZER,
+  LIST_INITIAL_VALUE(g_vhost_bus.device),
+  LIST_INITIAL_VALUE(g_vhost_bus.defered_device),
+  LIST_INITIAL_VALUE(g_vhost_bus.driver),
+};
+
+/****************************************************************************
+ * Private Functions
+ ****************************************************************************/
+
+/****************************************************************************
+ * Name: vhost_status_driver_ok
+ ****************************************************************************/
+
+static bool vhost_status_driver_ok(FAR struct vhost_device *hdev)
+{
+  uint8_t status = vhost_get_status(hdev);
+  bool driver_ok = false;
+
+  /* Busy wait until the remote is ready */
+
+  if (status & VIRTIO_CONFIG_STATUS_NEEDS_RESET)
+    {
+      vhost_set_status(hdev, 0);
+    }
+  else if (status & VIRTIO_CONFIG_STATUS_DRIVER_OK)
+    {
+      driver_ok = true;
+    }
+
+  return driver_ok;
+}
+
+/****************************************************************************
+ * Name: vhost_defered_probe_work
+ ****************************************************************************/
+
+static void vhost_defered_probe_work(FAR void *arg)
+{
+  FAR struct vhost_device_item_s *item;
+  FAR struct vhost_device_item_s *tmp;
+  FAR struct vhost_driver *driver;
+
+  nxmutex_lock(&g_vhost_bus.lock);
+
+  list_for_every_entry_safe(&g_vhost_bus.defered_device, item, tmp,
+                            struct vhost_device_item_s, node)
+    {
+      if (!vhost_status_driver_ok(item->device))
+        {
+          vhosterr("device is not ok device=%p\n", item->device);
+          continue;
+        }
+
+      /* Vhost device (virtio driver) status has been OK, move it to the
+       * normal device list
+       */
+
+      list_delete(&item->node);
+      list_add_tail(&g_vhost_bus.device, &item->node);
+      list_for_every_entry(&g_vhost_bus.driver, driver,
+                           struct vhost_driver, node)
+        {
+          if (item->device->id.device == driver->device)
+            {
+              if (driver->probe(item->device) >= 0)
+                {
+                  vhosterr("device probe success device=%p\n", item->device);
+                  item->driver = driver;
+                }
+
+              break;
+            }
+        }
+    }
+
+  if (!list_is_empty(&g_vhost_bus.defered_device))
+    {
+      work_queue(LPWORK, &g_vhost_bus.defered_work, vhost_defered_probe_work,
+                 NULL, VHOST_DEFERED_PROBE_PERIOD);
+    }
+
+  nxmutex_unlock(&g_vhost_bus.lock);
+}
+
+/****************************************************************************
+ * Name: vhost_remove_device
+ ****************************************************************************/
+
+static void vhost_remove_device(FAR struct vhost_device_item_s *item)
+{
+  /* Call driver remove and mark item->driver NULL to indicate
+   * the device unmatched.
+   */
+
+  item->driver->remove(item->device);
+  item->driver = NULL;
+
+  /* Remove the device from the device list and free memory */
+
+  list_delete(&item->node);
+  kmm_free(item);
+}
+
+/****************************************************************************
+ * Public Functions
+ ****************************************************************************/
+
+/****************************************************************************
+ * Name: vhost_register_driver
+ ****************************************************************************/
+
+int vhost_register_driver(FAR struct vhost_driver *driver)
+{
+  FAR struct vhost_device_item_s *item;
+  int ret;
+
+  DEBUGASSERT(driver != NULL && driver->probe != NULL &&
+              driver->remove != NULL);
+
+  ret = nxmutex_lock(&g_vhost_bus.lock);
+  if (ret < 0)
+    {
+      return ret;
+    }
+
+  /* Add the driver to the vhost_bus driver list */
+
+  list_add_tail(&g_vhost_bus.driver, &driver->node);
+
+  /* Match all the devices has registered in the vhost_bus */
+
+  list_for_every_entry(&g_vhost_bus.device, item, struct vhost_device_item_s,
+                       node)
+    {
+      if (driver->device == item->device->id.device)
+        {
+          /* If found the device in the device list, call driver probe,
+           * if probe success, assign item->driver to indicate the device
+           * matched.
+           */
+
+          if (driver->probe(item->device) >= 0)
+            {
+              item->driver = driver;
+            }
+        }
+    }
+
+  nxmutex_unlock(&g_vhost_bus.lock);
+  return ret;
+}
+
+/****************************************************************************
+ * Name: vhost_unregister_driver
+ ****************************************************************************/
+
+int vhost_unregister_driver(FAR struct vhost_driver *driver)
+{
+  FAR struct vhost_device_item_s *item;
+  int ret;
+
+  DEBUGASSERT(driver != NULL);
+
+  ret = nxmutex_lock(&g_vhost_bus.lock);
+  if (ret < 0)
+    {
+      return ret;
+    }
+
+  /* Find all the devices matched with driver in device list */
+
+  list_for_every_entry(&g_vhost_bus.device, item, struct vhost_device_item_s,
+                       node)
+    {
+      if (item->driver == driver)
+        {
+          /* 1. Call driver remove function;
+           * 2. Mark item->driver NULL to indicate the device unmatched;
+           */
+
+          driver->remove(item->device);
+          item->driver = NULL;
+        }
+    }
+
+  /* Remove the driver from the driver list */
+
+  list_delete(&driver->node);
+
+  nxmutex_unlock(&g_vhost_bus.lock);
+  return ret;
+}
+
+/****************************************************************************
+ * Name: vhost_register_device
+ ****************************************************************************/
+
+int vhost_register_device(FAR struct vhost_device *device)
+{
+  FAR struct vhost_device_item_s *item;
+  FAR struct vhost_driver *driver;
+  int ret;
+
+  item = kmm_zalloc(sizeof(*item));
+  if (item == NULL)
+    {
+      return -ENOMEM;
+    }
+
+  item->device = device;
+
+  ret = nxmutex_lock(&g_vhost_bus.lock);
+  if (ret < 0)
+    {
+      kmm_free(item);
+      return ret;
+    }
+
+  /* 1. Add device to defered device list if virtio driver not OK;
+   * 2. Add device to the normal device list and try to probe the driver
+   *    if virtio driver has been OK.
+   */
+
+  if (!vhost_status_driver_ok(device))
+    {
+      list_add_tail(&g_vhost_bus.defered_device, &item->node);
+      if (list_is_singular(&g_vhost_bus.defered_device))
+        {
+          work_queue(LPWORK, &g_vhost_bus.defered_work,
+                      vhost_defered_probe_work, NULL,
+                      VHOST_DEFERED_PROBE_PERIOD);
+        }
+    }
+  else
+    {
+      list_add_tail(&g_vhost_bus.device, &item->node);
+
+      /* Match the driver has registered in the vhost_bus */
+
+      list_for_every_entry(&g_vhost_bus.driver, driver, struct vhost_driver,
+                           node)
+        {
+          if (driver->device == device->id.device)
+            {
+              /* If found the driver in the driver list, call driver probe,
+               * if probe success, assign item->driver to indicate the device
+               * matched.
+               */
+
+              if (driver->probe(device) >= 0)
+                {
+                  item->driver = driver;
+                }
+
+              break;
+            }
+        }
+    }
+
+  nxmutex_unlock(&g_vhost_bus.lock);
+  return ret;
+}
+
+/****************************************************************************
+ * Name: vhost_unregister_device
+ ****************************************************************************/
+
+int vhost_unregister_device(FAR struct vhost_device *device)
+{
+  FAR struct vhost_device_item_s *item;
+  int ret;
+
+  ret = nxmutex_lock(&g_vhost_bus.lock);
+  if (ret < 0)
+    {
+      return ret;
+    }
+
+  /* Find the device in device list */
+
+  list_for_every_entry(&g_vhost_bus.device, item,
+                       struct vhost_device_item_s, node)
+    {
+      if (item->device == device)
+        {
+          vhost_remove_device(item);
+          goto out;
+        }
+    }
+
+  list_for_every_entry(&g_vhost_bus.defered_device, item,
+                       struct vhost_device_item_s, node)
+    {
+      if (item->device == device)
+        {
+          vhost_remove_device(item);
+          goto out;
+        }
+    }
+
+out:
+  nxmutex_unlock(&g_vhost_bus.lock);
+  return ret;
+}
+
+/****************************************************************************
+ * Name: vhost_register_drivers
+ ****************************************************************************/
+
+void vhost_register_drivers(void)
+{
+}

include/debug.h

@@ -920,6 +920,24 @@
 #  define vrtinfo     _none
 #endif
 
+#ifdef CONFIG_DEBUG_VHOST_ERROR
+#  define vhosterr     _err
+#else
+#  define vhosterr     _none
+#endif
+
+#ifdef CONFIG_DEBUG_VHOST_WARN
+#  define vhostwarn    _warn
+#else
+#  define vhostwarn    _none
+#endif
+
+#ifdef CONFIG_DEBUG_VHOST_INFO
+#  define vhostinfo    _info
+#else
+#  define vhostinfo    _none
+#endif
+
 #ifdef CONFIG_DEBUG_RESET_ERROR
 #  define rsterr       _err
 #else

include/nuttx/vhost/vhost.h

@@ -0,0 +1,98 @@
+/****************************************************************************
+ * include/nuttx/vhost/vhost.h
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.  The
+ * ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ ****************************************************************************/
+
+#ifndef __INCLUDE_NUTTX_VHOST_VHOST_H
+#define __INCLUDE_NUTTX_VHOST_VHOST_H
+
+/****************************************************************************
+ * Included Files
+ ****************************************************************************/
+
+#include <stdbool.h>
+#include <stdint.h>
+
+#ifdef CONFIG_DRIVERS_VHOST
+
+#include <nuttx/compiler.h>
+#include <nuttx/list.h>
+#include <openamp/open_amp.h>
+
+/****************************************************************************
+ * Pre-processor Definitions
+ ****************************************************************************/
+
+/****************************************************************************
+ * Public Type Definitions
+ ****************************************************************************/
+
+/* Wrapper the vhost API to virtio API */
+
+#define vhost_create_virtqueues virtio_create_virtqueues
+#define vhost_delete_virtqueues virtio_delete_virtqueues
+#define vhost_set_status        virtio_set_status
+#define vhost_get_status        virtio_get_status
+#define vhost_set_features      virtio_set_features
+#define vhost_get_features      virtio_get_features
+#define vhost_read_config       virtio_read_config
+#define vhost_write_config      virtio_write_config
+
+/* Wrapper the struct vhost_device to struct virtio_device */
+
+#define vhost_device            virtio_device
+
+struct vhost_driver
+{
+  struct list_node   node;
+  uint32_t           device;   /* device id */
+  CODE int         (*probe)(FAR struct vhost_device *hdev);
+  CODE void        (*remove)(FAR struct vhost_device *hdev);
+};
+
+/****************************************************************************
+ * Public Function Prototypes
+ ****************************************************************************/
+
+#ifdef __cplusplus
+#define EXTERN extern "C"
+extern "C"
+{
+#else
+#define EXTERN extern
+#endif
+
+int vhost_register_device(FAR struct vhost_device *hdev);
+int vhost_register_driver(FAR struct vhost_driver *hdrv);
+int vhost_unregister_driver(FAR struct vhost_driver *hdrv);
+int vhost_unregister_device(FAR struct vhost_device *hdev);
+
+/****************************************************************************
+ * Name: vhost_register_drivers
+ ****************************************************************************/
+
+void vhost_register_drivers(void);
+
+#undef EXTERN
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* CONFIG_DRIVERS_VHOST */
+
+#endif /* __INCLUDE_NUTTX_VHOST_VHOST_H */