[PATCH v2 06/15] example/vdpa:add vdpa blk support in example

[Andy Pei <andy.pei@intel.com>]

Add virtio blk device suppoort to vdpa example.

Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 examples/vdpa/Makefile           |   2 +-
 examples/vdpa/main.c             |   8 ++
 examples/vdpa/meson.build        |   1 +
 examples/vdpa/vdpa_blk_compact.c | 150 +++++++++++++++++++++++++++++++
 examples/vdpa/vdpa_blk_compact.h | 117 ++++++++++++++++++++++++
 examples/vdpa/vhost_user.h       | 189 +++++++++++++++++++++++++++++++++++++++
 6 files changed, 466 insertions(+), 1 deletion(-)
 create mode 100644 examples/vdpa/vdpa_blk_compact.c
 create mode 100644 examples/vdpa/vdpa_blk_compact.h
 create mode 100644 examples/vdpa/vhost_user.h

diff --git a/examples/vdpa/Makefile b/examples/vdpa/Makefile
index d974db4..9d0479b 100644
--- a/examples/vdpa/Makefile
+++ b/examples/vdpa/Makefile
@@ -5,7 +5,7 @@
 APP = vdpa
 
 # all source are stored in SRCS-y
-SRCS-y := main.c
+SRCS-y := main.c vdpa_blk_compact.c
 CFLAGS += -DALLOW_EXPERIMENTAL_API
 
 PKGCONF ?= pkg-config
diff --git a/examples/vdpa/main.c b/examples/vdpa/main.c
index 5ab0765..924ad7b 100644
--- a/examples/vdpa/main.c
+++ b/examples/vdpa/main.c
@@ -20,6 +20,7 @@
 #include <cmdline_parse_string.h>
 #include <cmdline_parse_num.h>
 #include <cmdline.h>
+#include "vdpa_blk_compact.h"
 
 #define MAX_PATH_LEN 128
 #define MAX_VDPA_SAMPLE_PORTS 1024
@@ -156,6 +157,7 @@ struct vdpa_port {
 static const struct rte_vhost_device_ops vdpa_sample_devops = {
 	.new_device = new_device,
 	.destroy_device = destroy_device,
+	.new_connection = rte_vhost_blk_session_install_rte_compat_hooks,
 };
 
 static int
@@ -192,6 +194,12 @@ struct vdpa_port {
 			"attach vdpa device failed: %s\n",
 			socket_path);
 
+	if (vdpa_blk_device_set_features_and_protocol(socket_path, vport->dev)
+		< 0)
+		rte_exit(EXIT_FAILURE,
+			"set vhost blk driver features and protocol features failed: %s\n",
+			socket_path);
+
 	if (rte_vhost_driver_start(socket_path) < 0)
 		rte_exit(EXIT_FAILURE,
 			"start vhost driver failed: %s\n",
diff --git a/examples/vdpa/meson.build b/examples/vdpa/meson.build
index bd08605..f0d111c 100644
--- a/examples/vdpa/meson.build
+++ b/examples/vdpa/meson.build
@@ -15,4 +15,5 @@ deps += 'vhost'
 allow_experimental_apis = true
 sources = files(
         'main.c',
+	'vdpa_blk_compact.c',
 )
diff --git a/examples/vdpa/vdpa_blk_compact.c b/examples/vdpa/vdpa_blk_compact.c
new file mode 100644
index 0000000..0c4d3ee
--- /dev/null
+++ b/examples/vdpa/vdpa_blk_compact.c
@@ -0,0 +1,150 @@
+/*    INTEL CONFIDENTIAL
+ *
+ *    Copyright (c) Intel Corporation.
+ *    All rights reserved.
+ *
+ *    The source code contained or described herein and all documents related
+ *    to the source code ("Material") are owned by Intel Corporation or its
+ *    suppliers or licensors.  Title to the Material remains with Intel
+ *    Corporation or its suppliers and licensors.  The Material contains trade
+ *    secrets and proprietary and confidential information of Intel or its
+ *    suppliers and licensors.  The Material is protected by worldwide
+ *    copyright and trade secret laws and treaty provisions.  No part of the
+ *    Material may be used, copied, reproduced, modified, published, uploaded,
+ *    posted, transmitted, distributed, or disclosed in any way without Intel's
+ *    prior express written permission.
+ *
+ *    No license under any patent, copyright, trade secret or other
+ *    intellectual property right is granted to or conferred upon you by
+ *    disclosure or delivery of the Materials, either expressly, by
+ *    implication, inducement, estoppel or otherwise.  Any license under such
+ *    intellectual property rights must be express and approved by Intel in
+ *    writing.
+ */
+
+/* @file
+ *
+ * Block device specific vhost lib
+ */
+
+#include <stdbool.h>
+
+#include <rte_malloc.h>
+#include <vdpa_driver.h>
+#include <rte_vhost.h>
+#include "vdpa_blk_compact.h"
+#include "vhost_user.h"
+
+#define VHOST_USER_GET_CONFIG	24
+#define VHOST_USER_SET_CONFIG	25
+
+#ifndef VHOST_USER_PROTOCOL_F_CONFIG
+#define VHOST_USER_PROTOCOL_F_CONFIG   9
+#endif
+
+/*
+ * Function to handle vhost user blk message
+ */
+static enum rte_vhost_msg_result
+rte_vhost_blk_extern_vhost_pre_msg_handler(int vid, void *_msg)
+{
+	struct VhostUserMsg *msg = _msg;
+	struct rte_vdpa_device *vdev = NULL;
+
+	vdev = rte_vhost_get_vdpa_device(vid);
+	if (vdev == NULL)
+		return RTE_VHOST_MSG_RESULT_ERR;
+
+	fprintf(stderr, "msg is %d\n", msg->request.master);
+	switch (msg->request.master) {
+	case VHOST_USER_GET_CONFIG: {
+		int rc = 0;
+
+		fprintf(stdout, "read message VHOST_USER_GET_CONFIG\n");
+
+		if (vdev->ops->get_config) {
+			fprintf(stdout, "get_config() function is valid!\n");
+			rc = vdev->ops->get_config(vid,
+						   msg->payload.cfg.region,
+						   msg->payload.cfg.size);
+			if (rc != 0) {
+				msg->size = 0;
+				fprintf(stdout, "get_config() return error!\n");
+			}
+		} else {
+			fprintf(stdout, "get_config() function is invalid!\n");
+		}
+
+		return RTE_VHOST_MSG_RESULT_REPLY;
+	}
+	case VHOST_USER_SET_CONFIG: {
+		int rc = 0;
+
+		fprintf(stdout,
+			"read message VHOST_USER_SET_CONFIG\n");
+
+		if (vdev->ops->set_config) {
+			rc = vdev->ops->set_config(vid,
+				msg->payload.cfg.region,
+				msg->payload.cfg.offset,
+				msg->payload.cfg.size,
+				msg->payload.cfg.flags);
+		}
+
+		return rc == 0 ? RTE_VHOST_MSG_RESULT_OK : RTE_VHOST_MSG_RESULT_ERR;
+	}
+	default:
+		break;
+	}
+
+	return RTE_VHOST_MSG_RESULT_NOT_HANDLED;
+}
+
+struct rte_vhost_user_extern_ops g_blk_extern_vhost_ops = {
+	.pre_msg_handle = rte_vhost_blk_extern_vhost_pre_msg_handler,
+	.post_msg_handle = NULL,
+};
+
+int
+rte_vhost_blk_session_install_rte_compat_hooks(int vid)
+{
+	int rc;
+
+	rc = rte_vhost_extern_callback_register(vid,
+						&g_blk_extern_vhost_ops,
+						NULL);
+	if (rc != 0) {
+		fprintf(stderr, "%s() failed for vid = %d\n",  __func__, vid);
+		return -1;
+	}
+	fprintf(stdout, "register extern vhost ops on vid = %d\n", vid);
+	return 0;
+}
+
+
+int
+vdpa_blk_device_set_features_and_protocol(const char *path,
+	struct rte_vdpa_device *vdev)
+{
+	uint64_t protocol_features = 0;
+
+	if (!vdev) {
+		fprintf(stdout, "vdev is NULL.\n");
+		return -EINVAL;
+	}
+
+	/* vdpa net does not have the get_config */
+	if (!vdev->ops->get_config)
+		return 0;
+
+	rte_vhost_driver_set_features(path, SPDK_VHOST_BLK_FEATURES_BASE);
+	rte_vhost_driver_disable_features(path,
+		SPDK_VHOST_BLK_DISABLED_FEATURES);
+
+	rte_vhost_driver_get_protocol_features(path, &protocol_features);
+	protocol_features |= (1ULL << VHOST_USER_PROTOCOL_F_CONFIG);
+	protocol_features |= (1ULL << VHOST_USER_PROTOCOL_F_LOG_SHMFD);
+	rte_vhost_driver_set_protocol_features(path, protocol_features);
+
+	return 0;
+}
diff --git a/examples/vdpa/vdpa_blk_compact.h b/examples/vdpa/vdpa_blk_compact.h
new file mode 100644
index 0000000..420d48e
--- /dev/null
+++ b/examples/vdpa/vdpa_blk_compact.h
@@ -0,0 +1,117 @@
+/*    INTEL CONFIDENTIAL
+ *
+ *    Copyright (c) Intel Corporation.
+ *    All rights reserved.
+ *
+ *    The source code contained or described herein and all documents related
+ *    to the source code ("Material") are owned by Intel Corporation or its
+ *    suppliers or licensors.  Title to the Material remains with Intel
+ *    Corporation or its suppliers and licensors.  The Material contains trade
+ *    secrets and proprietary and confidential information of Intel or its
+ *    suppliers and licensors.  The Material is protected by worldwide
+ *    copyright and trade secret laws and treaty provisions.  No part of the
+ *    Material may be used, copied, reproduced, modified, published, uploaded,
+ *    posted, transmitted, distributed, or disclosed in any way without Intel's
+ *    prior express written permission.
+ *
+ *    No license under any patent, copyright, trade secret or other
+ *    intellectual property right is granted to or conferred upon you by
+ *    disclosure or delivery of the Materials, either expressly, by
+ *    implication, inducement, estoppel or otherwise.  Any license under such
+ *    intellectual property rights must be express and approved by Intel in
+ *    writing.
+ */
+
+#ifndef _VDPA_BLK_COMPACT_H_
+#define _VDPA_BLK_COMPACT_H_
+
+/**
+ * @file
+ *
+ * Device specific vhost lib
+ */
+
+#include <stdbool.h>
+
+#include <rte_pci.h>
+#include <rte_vhost.h>
+
+/* Feature bits */
+#define VIRTIO_BLK_F_SIZE_MAX     1    /* Indicates maximum segment size */
+#define VIRTIO_BLK_F_SEG_MAX      2    /* Indicates maximum # of segments */
+#define VIRTIO_BLK_F_GEOMETRY     4    /* Legacy geometry available  */
+#define VIRTIO_BLK_F_RO           5    /* Disk is read-only */
+#define VIRTIO_BLK_F_BLK_SIZE     6    /* Block size of disk is available */
+#define VIRTIO_BLK_F_TOPOLOGY     10   /* Topology information is available */
+#define VIRTIO_BLK_F_MQ           12   /* support more than one vq */
+#define VIRTIO_BLK_F_DISCARD      13   /* DISCARD is supported */
+#define VIRTIO_BLK_F_WRITE_ZEROES 14   /* WRITE ZEROES is supported */
+
+/* Legacy feature bits */
+#ifndef VIRTIO_BLK_NO_LEGACY
+#define VIRTIO_BLK_F_BARRIER      0    /* Does host support barriers? */
+#define VIRTIO_BLK_F_SCSI         7    /* Supports scsi command passthru */
+#define VIRTIO_BLK_F_FLUSH        9    /* Flush command supported */
+#define VIRTIO_BLK_F_CONFIG_WCE   11   /* Writeback mode available in config */
+
+/* Old (deprecated) name for VIRTIO_BLK_F_FLUSH. */
+#define VIRTIO_BLK_F_WCE VIRTIO_BLK_F_FLUSH
+#endif /* !VIRTIO_BLK_NO_LEGACY */
+
+#ifndef VHOST_USER_F_PROTOCOL_FEATURES
+#define VHOST_USER_F_PROTOCOL_FEATURES 30
+#endif
+
+#define SPDK_VHOST_FEATURES ((1ULL << VHOST_F_LOG_ALL) | \
+	(1ULL << VHOST_USER_F_PROTOCOL_FEATURES) | \
+	(1ULL << VIRTIO_F_VERSION_1) | \
+	(1ULL << VIRTIO_F_NOTIFY_ON_EMPTY) | \
+	(1ULL << VIRTIO_RING_F_EVENT_IDX) | \
+	(1ULL << VIRTIO_RING_F_INDIRECT_DESC))
+
+#define SPDK_VHOST_DISABLED_FEATURES ((1ULL << VIRTIO_RING_F_EVENT_IDX) | \
+	(1ULL << VIRTIO_F_NOTIFY_ON_EMPTY))
+
+#define SPDK_VHOST_BLK_FEATURES_BASE (SPDK_VHOST_FEATURES | \
+	(1ULL << VIRTIO_BLK_F_SIZE_MAX) | (1ULL << VIRTIO_BLK_F_SEG_MAX) | \
+	(1ULL << VIRTIO_BLK_F_GEOMETRY) | (1ULL << VIRTIO_BLK_F_BLK_SIZE) | \
+	(1ULL << VIRTIO_BLK_F_TOPOLOGY) | (1ULL << VIRTIO_BLK_F_BARRIER)  | \
+	(1ULL << VIRTIO_BLK_F_SCSI)     | (1ULL << VIRTIO_BLK_F_CONFIG_WCE) | \
+	(1ULL << VIRTIO_BLK_F_MQ))
+
+/* Not supported features */
+#define SPDK_VHOST_BLK_DISABLED_FEATURES (SPDK_VHOST_DISABLED_FEATURES | \
+	(1ULL << VIRTIO_BLK_F_GEOMETRY) | (1ULL << VIRTIO_BLK_F_CONFIG_WCE) | \
+	(1ULL << VIRTIO_BLK_F_BARRIER)  | (1ULL << VIRTIO_BLK_F_SCSI))
+
+/* Vhost-blk support protocol features */
+#define SPDK_VHOST_BLK_PROTOCOL_FEATURES \
+	((1ULL << VHOST_USER_PROTOCOL_F_CONFIG) | \
+	(1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD))
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * This function will set vhost user block
+ *
+ * @param path
+ *  socket path
+ */
+int
+vdpa_blk_device_set_features_and_protocol(const char *path,
+	struct rte_vdpa_device *vdev);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Install external hook to handle vhost user block message
+ *
+ * @param vid
+ *  vhost device id
+ */
+int
+rte_vhost_blk_session_install_rte_compat_hooks(int vid);
+
+#endif /* _VDPA_BLK_COMPACT_H_ */
diff --git a/examples/vdpa/vhost_user.h b/examples/vdpa/vhost_user.h
new file mode 100644
index 0000000..8b747d0
--- /dev/null
+++ b/examples/vdpa/vhost_user.h
@@ -0,0 +1,189 @@
+/*    INTEL CONFIDENTIAL
+ *
+ *    Copyright (c) Intel Corporation.
+ *    All rights reserved.
+ *
+ *    The source code contained or described herein and all documents related
+ *    to the source code ("Material") are owned by Intel Corporation or its
+ *    suppliers or licensors.  Title to the Material remains with Intel
+ *    Corporation or its suppliers and licensors.  The Material contains trade
+ *    secrets and proprietary and confidential information of Intel or its
+ *    suppliers and licensors.  The Material is protected by worldwide
+ *    copyright and trade secret laws and treaty provisions.  No part of the
+ *    Material may be used, copied, reproduced, modified, published, uploaded,
+ *    posted, transmitted, distributed, or disclosed in any way without Intel's
+ *    prior express written permission.
+ *
+ *    No license under any patent, copyright, trade secret or other
+ *    intellectual property right is granted to or conferred upon you by
+ *    disclosure or delivery of the Materials, either expressly, by
+ *    implication, inducement, estoppel or otherwise.  Any license under such
+ *    intellectual property rights must be express and approved by Intel in
+ *    writing.
+ */
+
+#ifndef _VHOST_NET_USER_H
+#define _VHOST_NET_USER_H
+
+#include <stdint.h>
+#include <linux/vhost.h>
+
+#include "rte_vhost.h"
+
+/* refer to hw/virtio/vhost-user.c */
+
+#define VHOST_MEMORY_MAX_NREGIONS 8
+
+#ifndef VHOST_USER_MAX_CONFIG_SIZE
+#define VHOST_USER_MAX_CONFIG_SIZE		256
+#endif
+
+#define VHOST_USER_PROTOCOL_FEATURES	((1ULL << VHOST_USER_PROTOCOL_F_MQ) | \
+			(1ULL << VHOST_USER_PROTOCOL_F_LOG_SHMFD) |\
+			(1ULL << VHOST_USER_PROTOCOL_F_RARP) | \
+			(1ULL << VHOST_USER_PROTOCOL_F_REPLY_ACK) | \
+			(1ULL << VHOST_USER_PROTOCOL_F_NET_MTU) | \
+			(1ULL << VHOST_USER_PROTOCOL_F_SLAVE_REQ) | \
+			(1ULL << VHOST_USER_PROTOCOL_F_CRYPTO_SESSION) | \
+			(1ULL << VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD) | \
+			(1ULL << VHOST_USER_PROTOCOL_F_HOST_NOTIFIER) | \
+			(1ULL << VHOST_USER_PROTOCOL_F_PAGEFAULT))
+
+typedef enum VhostUserRequest {
+	VHOST_USER_NONE = 0,
+	VHOST_USER_GET_FEATURES = 1,
+	VHOST_USER_SET_FEATURES = 2,
+	VHOST_USER_SET_OWNER = 3,
+	VHOST_USER_RESET_OWNER = 4,
+	VHOST_USER_SET_MEM_TABLE = 5,
+	VHOST_USER_SET_LOG_BASE = 6,
+	VHOST_USER_SET_LOG_FD = 7,
+	VHOST_USER_SET_VRING_NUM = 8,
+	VHOST_USER_SET_VRING_ADDR = 9,
+	VHOST_USER_SET_VRING_BASE = 10,
+	VHOST_USER_GET_VRING_BASE = 11,
+	VHOST_USER_SET_VRING_KICK = 12,
+	VHOST_USER_SET_VRING_CALL = 13,
+	VHOST_USER_SET_VRING_ERR = 14,
+	VHOST_USER_GET_PROTOCOL_FEATURES = 15,
+	VHOST_USER_SET_PROTOCOL_FEATURES = 16,
+	VHOST_USER_GET_QUEUE_NUM = 17,
+	VHOST_USER_SET_VRING_ENABLE = 18,
+	VHOST_USER_SEND_RARP = 19,
+	VHOST_USER_NET_SET_MTU = 20,
+	VHOST_USER_SET_SLAVE_REQ_FD = 21,
+	VHOST_USER_IOTLB_MSG = 22,
+	VHOST_USER_CRYPTO_CREATE_SESS = 26,
+	VHOST_USER_CRYPTO_CLOSE_SESS = 27,
+	VHOST_USER_POSTCOPY_ADVISE = 28,
+	VHOST_USER_POSTCOPY_LISTEN = 29,
+	VHOST_USER_POSTCOPY_END = 30,
+	VHOST_USER_GET_INFLIGHT_FD = 31,
+	VHOST_USER_SET_INFLIGHT_FD = 32,
+	VHOST_USER_MAX = 33
+} VhostUserRequest;
+
+typedef enum VhostUserSlaveRequest {
+	VHOST_USER_SLAVE_NONE = 0,
+	VHOST_USER_SLAVE_IOTLB_MSG = 1,
+	VHOST_USER_SLAVE_VRING_HOST_NOTIFIER_MSG = 3,
+	VHOST_USER_SLAVE_MAX
+} VhostUserSlaveRequest;
+
+typedef struct VhostUserMemoryRegion {
+	uint64_t guest_phys_addr;
+	uint64_t memory_size;
+	uint64_t userspace_addr;
+	uint64_t mmap_offset;
+} VhostUserMemoryRegion;
+
+typedef struct VhostUserMemory {
+	uint32_t nregions;
+	uint32_t padding;
+	VhostUserMemoryRegion regions[VHOST_MEMORY_MAX_NREGIONS];
+} VhostUserMemory;
+
+typedef struct VhostUserLog {
+	uint64_t mmap_size;
+	uint64_t mmap_offset;
+} VhostUserLog;
+
+/* Comply with Cryptodev-Linux */
+#define VHOST_USER_CRYPTO_MAX_HMAC_KEY_LENGTH	512
+#define VHOST_USER_CRYPTO_MAX_CIPHER_KEY_LENGTH	64
+
+/* Same structure as vhost-user backend session info */
+typedef struct VhostUserCryptoSessionParam {
+	int64_t session_id;
+	uint32_t op_code;
+	uint32_t cipher_algo;
+	uint32_t cipher_key_len;
+	uint32_t hash_algo;
+	uint32_t digest_len;
+	uint32_t auth_key_len;
+	uint32_t aad_len;
+	uint8_t op_type;
+	uint8_t dir;
+	uint8_t hash_mode;
+	uint8_t chaining_dir;
+	uint8_t *ciphe_key;
+	uint8_t *auth_key;
+	uint8_t cipher_key_buf[VHOST_USER_CRYPTO_MAX_CIPHER_KEY_LENGTH];
+	uint8_t auth_key_buf[VHOST_USER_CRYPTO_MAX_HMAC_KEY_LENGTH];
+} VhostUserCryptoSessionParam;
+
+typedef struct VhostUserVringArea {
+	uint64_t u64;
+	uint64_t size;
+	uint64_t offset;
+} VhostUserVringArea;
+
+typedef struct VhostUserInflight {
+	uint64_t mmap_size;
+	uint64_t mmap_offset;
+	uint16_t num_queues;
+	uint16_t queue_size;
+} VhostUserInflight;
+
+/** Get/set config msg payload */
+struct vhost_user_config {
+	uint32_t offset;
+	uint32_t size;
+	uint32_t flags;
+	uint8_t region[VHOST_USER_MAX_CONFIG_SIZE];
+};
+
+typedef struct VhostUserMsg {
+	union {
+		uint32_t master; /* a VhostUserRequest value */
+		uint32_t slave;  /* a VhostUserSlaveRequest value*/
+	} request;
+
+#define VHOST_USER_VERSION_MASK     0x3
+#define VHOST_USER_REPLY_MASK       (0x1 << 2)
+#define VHOST_USER_NEED_REPLY		(0x1 << 3)
+	uint32_t flags;
+	uint32_t size; /* the following payload size */
+	union {
+#define VHOST_USER_VRING_IDX_MASK   0xff
+#define VHOST_USER_VRING_NOFD_MASK  (0x1<<8)
+		uint64_t u64;
+		struct vhost_vring_state state;
+		struct vhost_vring_addr addr;
+		VhostUserMemory memory;
+		VhostUserLog    log;
+		struct vhost_iotlb_msg iotlb;
+		VhostUserCryptoSessionParam crypto_session;
+		VhostUserVringArea area;
+		VhostUserInflight inflight;
+		struct vhost_user_config cfg;
+	} payload;
+	int fds[VHOST_MEMORY_MAX_NREGIONS];
+	int fd_num;
+} __attribute((packed)) VhostUserMsg;
+
+#define VHOST_USER_HDR_SIZE offsetof(VhostUserMsg, payload.u64)
+
+/* The version of the protocol we support */
+#define VHOST_USER_VERSION    0x1
+#endif
-- 
1.8.3.1