[dpdk-dev] [RFC PATCH 2/2] lib/librte_vhost: Add device abstraction layer

[Tetsuya Mukawa <mukawa@igel.co.jp>]

---
 lib/librte_vhost/Makefile                     |   6 +-
 lib/librte_vhost/rte_virtio_net.h             |  22 ++++-
 lib/librte_vhost/vhost-cuse/vhost-net-cdev.c  |   6 +-
 lib/librte_vhost/vhost-cuse/vhost-net-cdev.h  |  40 +++++++++
 lib/librte_vhost/vhost-cuse/virtio-net-cdev.c |   1 +
 lib/librte_vhost/vhost-net.c                  | 101 +++++++++++++++++++++++
 lib/librte_vhost/vhost-net.h                  | 114 ++++++++++++++++++++++++++
 lib/librte_vhost/vhost-user/vhost-net-user.c  |   6 +-
 lib/librte_vhost/vhost-user/vhost-net-user.h  |   3 +
 lib/librte_vhost/vhost-user/virtio-net-user.c |   1 +
 10 files changed, 290 insertions(+), 10 deletions(-)
 create mode 100644 lib/librte_vhost/vhost-cuse/vhost-net-cdev.h
 create mode 100644 lib/librte_vhost/vhost-net.c
 create mode 100644 lib/librte_vhost/vhost-net.h

diff --git a/lib/librte_vhost/Makefile b/lib/librte_vhost/Makefile
index cb4e172..4363a14 100644
--- a/lib/librte_vhost/Makefile
+++ b/lib/librte_vhost/Makefile
@@ -37,11 +37,11 @@ LIB = librte_vhost.a
 CFLAGS += $(WERROR_FLAGS) -I$(SRCDIR) -I. -I vhost-user -I vhost-cuse -O3 -D_FILE_OFFSET_BITS=64 -lfuse
 LDFLAGS += -lfuse
 # all source are stored in SRCS-y
-#SRCS-$(CONFIG_RTE_LIBRTE_VHOST) := vhost-cuse/vhost-net-cdev.c vhost-cuse/virtio-net-cdev.c
+SRCS-$(CONFIG_RTE_LIBRTE_VHOST) := vhost-cuse/vhost-net-cdev.c vhost-cuse/virtio-net-cdev.c
 
-SRCS-$(CONFIG_RTE_LIBRTE_VHOST) := vhost-user/fd_man.c vhost-user/vhost-net-user.c vhost-user/virtio-net-user.c
+SRCS-$(CONFIG_RTE_LIBRTE_VHOST) += vhost-user/fd_man.c vhost-user/vhost-net-user.c vhost-user/virtio-net-user.c
 
-SRCS-$(CONFIG_RTE_LIBRTE_VHOST) += virtio-net.c vhost_rxtx.c
+SRCS-$(CONFIG_RTE_LIBRTE_VHOST) += virtio-net.c vhost_rxtx.c vhost-net.c
 
 # install includes
 SYMLINK-$(CONFIG_RTE_LIBRTE_VHOST)-include += rte_virtio_net.h
diff --git a/lib/librte_vhost/rte_virtio_net.h b/lib/librte_vhost/rte_virtio_net.h
index a09533d..116c7e9 100644
--- a/lib/librte_vhost/rte_virtio_net.h
+++ b/lib/librte_vhost/rte_virtio_net.h
@@ -140,6 +140,23 @@ gpa_to_vva(struct virtio_net *dev, uint64_t guest_pa)
 }
 
 /**
+ * Enum for vhost driver types.
+ */
+enum rte_vhost_driver_t {
+       VHOST_DRV_CUSE, /* vhost-cuse driver */
+       VHOST_DRV_USER, /* vhost-user driver */
+       VHOST_DRV_NUM   /* the number of vhost driver types */
+};
+
+/**
+  * Structure contains information relating vhost driver.
+  */
+struct rte_vhost_driver {
+	enum rte_vhost_driver_t type;		/**< driver type. */
+	const char		*dev_name;	/**< accessing device name. */
+};
+
+/**
  * Disable features in feature_mask. Returns 0 on success.
  */
 int rte_vhost_feature_disable(uint64_t feature_mask);
@@ -155,12 +172,13 @@ uint64_t rte_vhost_feature_get(void);
 int rte_vhost_enable_guest_notification(struct virtio_net *dev, uint16_t queue_id, int enable);
 
 /* Register vhost driver. dev_name could be different for multiple instance support. */
-int rte_vhost_driver_register(const char *dev_name);
+struct rte_vhost_driver *rte_vhost_driver_register(
+		const char *dev_name, enum rte_vhost_driver_t type);
 
 /* Register callbacks. */
 int rte_vhost_driver_callback_register(struct virtio_net_device_ops const * const);
 
-int rte_vhost_driver_session_start(void);
+int rte_vhost_driver_session_start(struct rte_vhost_driver *drv);
 
 /**
  * This function adds buffers to the virtio devices RX virtqueue. Buffers can
diff --git a/lib/librte_vhost/vhost-cuse/vhost-net-cdev.c b/lib/librte_vhost/vhost-cuse/vhost-net-cdev.c
index 688ec00..6ea54ee 100644
--- a/lib/librte_vhost/vhost-cuse/vhost-net-cdev.c
+++ b/lib/librte_vhost/vhost-cuse/vhost-net-cdev.c
@@ -47,6 +47,7 @@
 
 #include "virtio-net-cdev.h"
 #include "vhost-net.h"
+#include "vhost-net-cdev.h"
 #include "eventfd_link/eventfd_link.h"
 
 #define FUSE_OPT_DUMMY "\0\0"
@@ -373,8 +374,9 @@ static const struct cuse_lowlevel_ops vhost_net_ops = {
  * vhost_net_device_ops are also passed when the device is registered in app.
  */
 int
-rte_vhost_driver_register(const char *dev_name)
+vhost_cuse_driver_register(struct rte_vhost_driver *drv)
 {
+	const char *dev_name = drv->dev_name;
 	struct cuse_info cuse_info;
 	char device_name[PATH_MAX] = "";
 	char char_device_name[PATH_MAX] = "";
@@ -428,7 +430,7 @@ rte_vhost_driver_register(const char *dev_name)
  * release and ioctl calls.
  */
 int
-rte_vhost_driver_session_start(void)
+vhost_cuse_driver_session_start(void)
 {
 	fuse_session_loop(session);
 
diff --git a/lib/librte_vhost/vhost-cuse/vhost-net-cdev.h b/lib/librte_vhost/vhost-cuse/vhost-net-cdev.h
new file mode 100644
index 0000000..cb094ee
--- /dev/null
+++ b/lib/librte_vhost/vhost-cuse/vhost-net-cdev.h
@@ -0,0 +1,40 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2014 IGEL Co.,Ltd. All rights reserved.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of IGEL nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _VHOST_NET_CDEV_H
+#define _VHOST_NET_CDEV_H
+
+int vhost_cuse_driver_register(struct rte_vhost_driver *drv);
+int vhost_cuse_driver_session_start(void);
+
+#endif
diff --git a/lib/librte_vhost/vhost-cuse/virtio-net-cdev.c b/lib/librte_vhost/vhost-cuse/virtio-net-cdev.c
index 7381140..42a6b24 100644
--- a/lib/librte_vhost/vhost-cuse/virtio-net-cdev.c
+++ b/lib/librte_vhost/vhost-cuse/virtio-net-cdev.c
@@ -46,6 +46,7 @@
 #include <errno.h>
 
 #include <rte_log.h>
+#include <rte_virtio_net.h>
 
 #include "vhost-net.h"
 #include "virtio-net-cdev.h"
diff --git a/lib/librte_vhost/vhost-net.c b/lib/librte_vhost/vhost-net.c
new file mode 100644
index 0000000..7a4537d
--- /dev/null
+++ b/lib/librte_vhost/vhost-net.c
@@ -0,0 +1,101 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2014 IGEL Co.,Ltd. All rights reserved.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of IGEL nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <errno.h>
+#include <rte_malloc.h>
+#include <rte_virtio_net.h>
+
+#include "vhost-cuse/vhost-net-cdev.h"
+#include "vhost-user/vhost-net-user.h"
+
+/**
+ * This function abstracts cuse and vhost-user driver registration.
+ */
+struct rte_vhost_driver *
+rte_vhost_driver_register(const char *dev_name, enum rte_vhost_driver_t type)
+{
+	int ret;
+	struct rte_vhost_driver *drv;
+
+	drv = rte_zmalloc(dev_name, sizeof(struct rte_vhost_driver),
+			CACHE_LINE_SIZE);
+	if (drv == NULL)
+		return NULL;
+
+	drv->dev_name = dev_name;
+	drv->type = type;
+
+	switch (type) {
+	case VHOST_DRV_CUSE:
+		ret = vhost_cuse_driver_register(drv);
+		if (ret != 0)
+			goto err;
+		break;
+	case VHOST_DRV_USER:
+		ret = vhost_user_driver_register(drv);
+		if (ret != 0)
+			goto err;
+		break;
+	default:
+		break;
+	}
+
+	return drv;
+err:
+	free(drv);
+	return NULL;
+}
+
+/**
+ * The session is launched allowing the application to
+ * receive open, release and ioctl calls.
+ */
+int
+rte_vhost_driver_session_start(struct rte_vhost_driver *drv)
+{
+	if (drv == NULL)
+		return -ENODEV;
+
+	switch (drv->type) {
+	case VHOST_DRV_CUSE:
+		vhost_cuse_driver_session_start();
+		break;
+	case VHOST_DRV_USER:
+		vhost_user_driver_session_start();
+		break;
+	default:
+		break;
+	}
+
+	return 0;
+}
diff --git a/lib/librte_vhost/vhost-net.h b/lib/librte_vhost/vhost-net.h
new file mode 100644
index 0000000..881a45f
--- /dev/null
+++ b/lib/librte_vhost/vhost-net.h
@@ -0,0 +1,114 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _VHOST_NET_H_
+#define _VHOST_NET_H_
+#include <stdint.h>
+#include <stdio.h>
+#include <sys/types.h>
+#include <unistd.h>
+#include <linux/vhost.h>
+
+#include <rte_log.h>
+
+/* Macros for printing using RTE_LOG */
+#define RTE_LOGTYPE_VHOST_CONFIG RTE_LOGTYPE_USER1
+#define RTE_LOGTYPE_VHOST_DATA   RTE_LOGTYPE_USER1
+
+#ifdef RTE_LIBRTE_VHOST_DEBUG
+#define VHOST_MAX_PRINT_BUFF 6072
+#define LOG_LEVEL RTE_LOG_DEBUG
+#define LOG_DEBUG(log_type, fmt, args...) RTE_LOG(DEBUG, log_type, fmt, ##args)
+#define VHOST_PRINT_PACKET(device, addr, size, header) do { \
+	char *pkt_addr = (char *)(addr); \
+	unsigned int index; \
+	char packet[VHOST_MAX_PRINT_BUFF]; \
+	\
+	if ((header)) \
+		snprintf(packet, VHOST_MAX_PRINT_BUFF, "(%"PRIu64") Header size %d: ", (device->device_fh), (size)); \
+	else \
+		snprintf(packet, VHOST_MAX_PRINT_BUFF, "(%"PRIu64") Packet size %d: ", (device->device_fh), (size)); \
+	for (index = 0; index < (size); index++) { \
+		snprintf(packet + strnlen(packet, VHOST_MAX_PRINT_BUFF), VHOST_MAX_PRINT_BUFF - strnlen(packet, VHOST_MAX_PRINT_BUFF), \
+			"%02hhx ", pkt_addr[index]); \
+	} \
+	snprintf(packet + strnlen(packet, VHOST_MAX_PRINT_BUFF), VHOST_MAX_PRINT_BUFF - strnlen(packet, VHOST_MAX_PRINT_BUFF), "\n"); \
+	\
+	LOG_DEBUG(VHOST_DATA, "%s", packet); \
+} while (0)
+#else
+#define LOG_LEVEL RTE_LOG_INFO
+#define LOG_DEBUG(log_type, fmt, args...) do {} while (0)
+#define VHOST_PRINT_PACKET(device, addr, size, header) do {} while (0)
+#endif
+
+
+/*
+ * Structure used to identify device context.
+ */
+struct vhost_device_ctx {
+	pid_t		pid;	/* PID of process calling the IOCTL. */
+	uint64_t	fh;	/* Populated with fi->fh to track the device index. */
+};
+
+/*
+ * Structure contains function pointers to be defined in virtio-net.c. These
+ * functions are called in CUSE context and are used to configure devices.
+ */
+struct vhost_net_device_ops {
+	int (*new_device)(struct vhost_device_ctx);
+	void (*destroy_device)(struct vhost_device_ctx);
+
+	int (*get_features)(struct vhost_device_ctx, uint64_t *);
+	int (*set_features)(struct vhost_device_ctx, uint64_t *);
+
+	int (*set_mem_table)(struct vhost_device_ctx,
+			const struct virtio_memory_regions *, uint32_t);
+
+	int (*set_vring_num)(struct vhost_device_ctx, struct vhost_vring_state *);
+	int (*set_vring_addr)(struct vhost_device_ctx, struct vhost_vring_addr *);
+	int (*set_vring_base)(struct vhost_device_ctx, struct vhost_vring_state *);
+	int (*get_vring_base)(struct vhost_device_ctx, uint32_t, struct vhost_vring_state *);
+
+	int (*set_vring_kick)(struct vhost_device_ctx, struct vhost_vring_file *);
+	int (*set_vring_call)(struct vhost_device_ctx, struct vhost_vring_file *);
+
+	int (*set_backend)(struct vhost_device_ctx, struct vhost_vring_file *);
+
+	int (*set_owner)(struct vhost_device_ctx);
+	int (*reset_owner)(struct vhost_device_ctx);
+};
+
+
+struct vhost_net_device_ops const *get_virtio_net_callbacks(void);
+#endif /* _VHOST_NET_H_ */
diff --git a/lib/librte_vhost/vhost-user/vhost-net-user.c b/lib/librte_vhost/vhost-user/vhost-net-user.c
index 0b100ba..837f840 100644
--- a/lib/librte_vhost/vhost-user/vhost-net-user.c
+++ b/lib/librte_vhost/vhost-user/vhost-net-user.c
@@ -371,9 +371,9 @@ vserver_message_handler(int connfd, uint64_t dat)
  * Creates and initialise the vhost server.
  */
 int
-rte_vhost_driver_register(const char *path)
+vhost_user_driver_register(struct rte_vhost_driver *drv)
 {
-
+	const char *path = drv->dev_name;
 	struct vhost_server *vserver;
 
 	if (g_vhost_server != NULL)
@@ -408,7 +408,7 @@ rte_vhost_driver_register(const char *path)
 
 
 int
-rte_vhost_driver_session_start(void)
+vhost_user_driver_session_start(void)
 {
 	fdset_event_dispatch(&g_vhost_server->fdset);
 	return 0;
diff --git a/lib/librte_vhost/vhost-user/vhost-net-user.h b/lib/librte_vhost/vhost-user/vhost-net-user.h
index c9df9fa..d90c147 100644
--- a/lib/librte_vhost/vhost-user/vhost-net-user.h
+++ b/lib/librte_vhost/vhost-user/vhost-net-user.h
@@ -71,4 +71,7 @@ typedef struct VhostUserMsg {
 #define VHOST_USER_VERSION    (0x1)
 
 /*****************************************************************************/
+int vhost_user_driver_register(struct rte_vhost_driver *drv);
+int vhost_user_driver_session_start(void);
+
 #endif
diff --git a/lib/librte_vhost/vhost-user/virtio-net-user.c b/lib/librte_vhost/vhost-user/virtio-net-user.c
index 4103977..f839219 100644
--- a/lib/librte_vhost/vhost-user/virtio-net-user.c
+++ b/lib/librte_vhost/vhost-user/virtio-net-user.c
@@ -38,6 +38,7 @@
 #include <sys/mman.h>
 
 #include <rte_log.h>
+#include <rte_virtio_net.h>
 
 #include "virtio-net-user.h"
 #include "vhost-net-user.h"
-- 
1.9.1